From pypy.commits at gmail.com Sat Jun 1 12:42:52 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sat, 01 Jun 2019 09:42:52 -0700 (PDT) Subject: [pypy-commit] pypy winconsoleio: Added testconsole, further implementation of winconsoleio. Message-ID: <5cf2ab0c.1c69fb81.2c092.6394@mx.google.com> Author: andrewjlawrence Branch: winconsoleio Changeset: r96725:64a78c073e0b Date: 2019-06-01 17:40 +0100 http://bitbucket.org/pypy/pypy/changeset/64a78c073e0b/ Log: Added testconsole, further implementation of winconsoleio. diff --git a/pypy/module/_io/interp_win32consoleio.py b/pypy/module/_io/interp_win32consoleio.py --- a/pypy/module/_io/interp_win32consoleio.py +++ b/pypy/module/_io/interp_win32consoleio.py @@ -1,25 +1,44 @@ import sys -from pypy.module._io.interp_iobase import W_RawIOBase +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec +from pypy.module._io.interp_iobase import (W_RawIOBase, DEFAULT_BUFFER_SIZE) from rpython.rlib import rwin32 +def _get_console_type(): + pass + def _pyio_get_console_type(): pass class W_WinConsoleIO(W_RawIOBase): + SMALLBUF = 4 + def __init__(self, space): W_RawIOBase.__init__(self, space) + self.handle = rwin32.INVALID_HANDLE_VALUE + self.fd = -1 + self.created = 0 + self.readable = 0 + self.writable = 0 + self.closehandle = 0 + self.blksize = 0 - def descr_init(self, space, w_nameobj, w_mode="r", w_closefd=True, w_opener=None): + @unwrap_spec(w_mode=WrappedDefault("r"), w_closefd=WrappedDefault(True), w_opener=WrappedDefault(None)) + def descr_init(self, space, w_nameobj, w_mode, w_closefd, w_opener): #self.fd = -1 #self.created = 0 self.readable = False self.writable = False #self.closehandle = 0; - #self.blksize = 0 + self.blksize = 0 rwa = False console_type = '\0' + self.buf = lltype.malloc(rffi.CCHARPP.TO,SMALLBUF,flavor='raw') + self.fd = space.int_w(w_nameobj) + closefd = space.bool_w(w_closefd) + if self.fd < 0: decodedname = space.fsdecode_w(w_nameobj) name = rffi.cast(rffi.CWCHARP, decodedname) @@ -36,7 +55,7 @@ if char in "+abx": # OK do nothing pass - else if char == "r": + elif char == "r": if rwa: raise oefmt(space.w_ValueError, "invalid mode: %.200s", mode) @@ -44,7 +63,7 @@ self.readable = True if console_type == "x": console_type = "r" - else if char == "w": + elif char == "w": if rwa: raise oefmt(space.w_ValueError, "invalid mode: %.200s", mode) @@ -64,3 +83,53 @@ self.closehandle = False else: access = rwin32.GENERIC_READ + self.closehandle = True + if not closefd: + raise oefmt(space.w_ValueError, + "Cannot use closefd=False with a file name") + if self.writeable: + access = rwin32.GENERIC_WRITE + + from rpython.rlib._os_support import _preferred_traits, string_trait + traits = _preferred_traits(name) + if not (traits.str is unicode): + raise oefmt(space.w_ValueError, + "Non-unicode string name %s", traits.str) + win32traits = make_win32_traits(traits) + self.handle = win32traits.CreateFile(name, + rwin32.GENERIC_READ | rwin32.GENERIC_WRITE, + rwin32.FILE_SHARE_READ | rwin32.FILE_SHARE_WRITE, + rffi.NULL, win32traits.OPEN_EXISTING, 0, rffi.NULL) + if self.handle == rwin32.INVALID_HANDLE_VALUE: + self.handle = win32traits.CreateFile(name, + access, + rwin32.FILE_SHARE_READ | rwin32.FILE_SHARE_WRITE, + rffi.NULL, win32traits.OPEN_EXISTING, 0, rffi.NULL) + + if self.handle == rwin32.INVALID_HANDLE_VALUE: + raise WindowsError(rwin32.GetLastError_saved(), + "Failed to open handle") + + if console_type == '\0': + console_type = _get_console_type(self.handle) + + if console_type == '\0': + raise oefmt(space.w_ValueError, + "Cannot open non-console file") + + if self.writable and console_type != 'w': + raise oefmt(space.w_ValueError, + "Cannot open input buffer for writing") + + if self.readable and console_type != 'r': + raise oefmt(space.w_ValueError, + "Cannot open output buffer for reading") + + self.blksize = DEFAULT_BUFFER_SIZE + rffi.c_memset(self.buf, 0, SMALLBUF) + +W_WinConsoleIO.typedef = TypeDef( + '_io.WinConsoleIO', W_WinConsoleIO.typedef, + #__new__ = interp2app(W_FileIO.descr_new.im_func), + __init__ = interp2app(W_WinConsoleIO.descr_init), + ) diff --git a/pypy/module/_testconsole/__init__.py b/pypy/module/_testconsole/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/module/_testconsole/__init__.py @@ -0,0 +1,11 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + + appleveldefs = { + } + + interpleveldefs = { + 'write_input': 'interp_testconsole.write_input', + 'read_output': 'interp_testconsole.read_output', + } diff --git a/pypy/module/_testconsole/interp_testconsole.py b/pypy/module/_testconsole/interp_testconsole.py new file mode 100644 --- /dev/null +++ b/pypy/module/_testconsole/interp_testconsole.py @@ -0,0 +1,10 @@ +import sys + +from pypy.module._io.interp_win32consoleio import W_WinConsoleIO +from rpython.rlib import rwin32 + +def write_input(): + pass + +def read_output(): + pass diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -137,10 +137,12 @@ HMODULE = HANDLE NULL_HANDLE = rffi.cast(HANDLE, 0) INVALID_HANDLE_VALUE = rffi.cast(HANDLE, -1) - GENERIC_READ = rffi.cast(DWORD, 0x80000000) - GENERIC_WRITE = rffi.cast(DWORD, 0x40000000) - GENERIC_EXECUTE= rffi.cast(DWORD, 0x20000000) - GENERIC_ALL = rffi.cast(DWORD, 0x10000000) + GENERIC_READ = rffi.cast(DWORD, 0x80000000) + GENERIC_WRITE = rffi.cast(DWORD, 0x40000000) + GENERIC_EXECUTE = rffi.cast(DWORD, 0x20000000) + GENERIC_ALL = rffi.cast(DWORD, 0x10000000) + FILE_SHARE_READ = rffi.cast(DWORD, 0x00000001) + FILE_SHARE_WRITE = rffi.cast(DWORD, 0x00000002) PFILETIME = rffi.CArrayPtr(FILETIME) From pypy.commits at gmail.com Sat Jun 1 17:49:55 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sat, 01 Jun 2019 14:49:55 -0700 (PDT) Subject: [pypy-commit] pypy winconsoleio: Work in progress. Message-ID: <5cf2f303.1c69fb81.89026.42a4@mx.google.com> Author: andrewjlawrence Branch: winconsoleio Changeset: r96726:462ddd6f090c Date: 2019-06-01 22:48 +0100 http://bitbucket.org/pypy/pypy/changeset/462ddd6f090c/ Log: Work in progress. diff --git a/pypy/module/_io/interp_win32consoleio.py b/pypy/module/_io/interp_win32consoleio.py --- a/pypy/module/_io/interp_win32consoleio.py +++ b/pypy/module/_io/interp_win32consoleio.py @@ -1,19 +1,47 @@ import sys +from pypy.interpreter.error import oefmt from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec from pypy.module._io.interp_iobase import (W_RawIOBase, DEFAULT_BUFFER_SIZE) +from pypy.interpreter.unicodehelper import fsdecode +from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib import rwin32 -def _get_console_type(): - pass +SMALLBUF = 4 -def _pyio_get_console_type(): - pass +def _get_console_type(handle): + try: + mode = lltype.malloc(rffi.DWORD,0,flavor='raw') + peek_count = lltype.malloc(rffi.DWORD,0,flavor='raw') + + if handle == INVALID_HANDLE_VALUE: + return '\0' + + if not rwin32.GetConsoleMode(handle, mode): + return '\0' + + # Peek at the handle to see whether it is an input or output handle + if rwin32.GetNumberOfConsoleInputEvents(handle, peek_count): + return 'r' + return 'w' + finally: + lltype.free(mode, flavor='raw') + lltype.free(peek_count, flavor='raw') + +def _pyio_get_console_type(path_or_fd): + fd = int(path_or_fd) + if fd >= 0: + handle = rwin32.get_osfhandle(fd) + if handle == rwin32.INVALID_HANDLE_VALUE: + return '\0' + return _get_console_type(handle) + + #if not fsdecode(path_or_fd, decoded): + # return '\0'; + class W_WinConsoleIO(W_RawIOBase): - SMALLBUF = 4 - def __init__(self, space): W_RawIOBase.__init__(self, space) self.handle = rwin32.INVALID_HANDLE_VALUE @@ -36,97 +64,100 @@ console_type = '\0' self.buf = lltype.malloc(rffi.CCHARPP.TO,SMALLBUF,flavor='raw') - self.fd = space.int_w(w_nameobj) - closefd = space.bool_w(w_closefd) + try: + self.fd = space.int_w(w_nameobj) + closefd = space.bool_w(w_closefd) - if self.fd < 0: - decodedname = space.fsdecode_w(w_nameobj) - name = rffi.cast(rffi.CWCHARP, decodedname) - console_type = _pyio_get_console_type(decodedname) - if not console_type: + if self.fd < 0: + decodedname = space.fsdecode_w(w_nameobj) + name = rffi.cast(rffi.CWCHARP, decodedname) + console_type = _pyio_get_console_type(decodedname) + if not console_type: + raise oefmt(space.w_ValueError, + "Invalid console type") + if console_type == '\0': + raise oefmt(space.w_ValueError, + "Cannot open non-console file") + s = space.text_w(w_mode) + + for char in s: + if char in "+abx": + # OK do nothing + pass + elif char == "r": + if rwa: + raise oefmt(space.w_ValueError, + "invalid mode: %s", space.text_w(w_mode)) + rwa = True + self.readable = True + if console_type == "x": + console_type = "r" + elif char == "w": + if rwa: + raise oefmt(space.w_ValueError, + "invalid mode: %s", space.text_w(w_mode)) + rwa = True + self.writable = True; + if console_type == 'x': + console_type = 'w' + else: + raise oefmt(space.w_ValueError, + "invalid mode: %s", space.text_w(w_mode)) + if not rwa: raise oefmt(space.w_ValueError, - "Invalid console type") - if console_type == '\0': - raise oefmt(space.w_ValueError, - "Cannot open non-console file") - s = space.text_w(w_mode) - - for char in s: - if char in "+abx": - # OK do nothing - pass - elif char == "r": - if rwa: + "Must have exactly one of read or write mode") + + if self.fd >= 0: + self.handle = rwin32.get_osfhandle(self.fd) + self.closehandle = False + else: + access = rwin32.GENERIC_READ + self.closehandle = True + if not closefd: raise oefmt(space.w_ValueError, - "invalid mode: %.200s", mode) - rwa = True - self.readable = True - if console_type == "x": - console_type = "r" - elif char == "w": - if rwa: + "Cannot use closefd=False with a file name") + if self.writeable: + access = rwin32.GENERIC_WRITE + + from rpython.rlib._os_support import _preferred_traits, string_trait + traits = _preferred_traits(name) + if not (traits.str is unicode): raise oefmt(space.w_ValueError, - "invalid mode: %.200s", mode) - rwa = True - self.writable = True; - if console_type == 'x': - console_type = 'w' - else: - raise oefmt(space.w_ValueError, - "invalid mode: %.200s", mode) - if not rwa: - raise oefmt(space.w_ValueError, - "Must have exactly one of read or write mode") - - if self.fd >= 0: - self.handle = rwin32.get_osfhandle(self.fd) - self.closehandle = False - else: - access = rwin32.GENERIC_READ - self.closehandle = True - if not closefd: - raise oefmt(space.w_ValueError, - "Cannot use closefd=False with a file name") - if self.writeable: - access = rwin32.GENERIC_WRITE - - from rpython.rlib._os_support import _preferred_traits, string_trait - traits = _preferred_traits(name) - if not (traits.str is unicode): - raise oefmt(space.w_ValueError, - "Non-unicode string name %s", traits.str) - win32traits = make_win32_traits(traits) - self.handle = win32traits.CreateFile(name, - rwin32.GENERIC_READ | rwin32.GENERIC_WRITE, - rwin32.FILE_SHARE_READ | rwin32.FILE_SHARE_WRITE, - rffi.NULL, win32traits.OPEN_EXISTING, 0, rffi.NULL) - if self.handle == rwin32.INVALID_HANDLE_VALUE: + "Non-unicode string name %s", traits.str) + win32traits = make_win32_traits(traits) self.handle = win32traits.CreateFile(name, - access, + rwin32.GENERIC_READ | rwin32.GENERIC_WRITE, rwin32.FILE_SHARE_READ | rwin32.FILE_SHARE_WRITE, rffi.NULL, win32traits.OPEN_EXISTING, 0, rffi.NULL) + if self.handle == rwin32.INVALID_HANDLE_VALUE: + self.handle = win32traits.CreateFile(name, + access, + rwin32.FILE_SHARE_READ | rwin32.FILE_SHARE_WRITE, + rffi.NULL, win32traits.OPEN_EXISTING, 0, rffi.NULL) - if self.handle == rwin32.INVALID_HANDLE_VALUE: - raise WindowsError(rwin32.GetLastError_saved(), - "Failed to open handle") - - if console_type == '\0': - console_type = _get_console_type(self.handle) + if self.handle == rwin32.INVALID_HANDLE_VALUE: + raise WindowsError(rwin32.GetLastError_saved(), + "Failed to open handle") + + if console_type == '\0': + console_type = _get_console_type(self.handle) - if console_type == '\0': - raise oefmt(space.w_ValueError, - "Cannot open non-console file") - - if self.writable and console_type != 'w': - raise oefmt(space.w_ValueError, - "Cannot open input buffer for writing") + if console_type == '\0': + raise oefmt(space.w_ValueError, + "Cannot open non-console file") + + if self.writable and console_type != 'w': + raise oefmt(space.w_ValueError, + "Cannot open input buffer for writing") - if self.readable and console_type != 'r': - raise oefmt(space.w_ValueError, - "Cannot open output buffer for reading") + if self.readable and console_type != 'r': + raise oefmt(space.w_ValueError, + "Cannot open output buffer for reading") - self.blksize = DEFAULT_BUFFER_SIZE - rffi.c_memset(self.buf, 0, SMALLBUF) + self.blksize = DEFAULT_BUFFER_SIZE + rffi.c_memset(self.buf, 0, SMALLBUF) + finally: + lltype.free(self.buf, flavor='raw') W_WinConsoleIO.typedef = TypeDef( '_io.WinConsoleIO', W_WinConsoleIO.typedef, diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -560,3 +560,9 @@ os.close(fd2) raise return res + + GetConsoleMode = winexternal( + 'GetConsoleMode', [HANDLE, LPDWORD], BOOL) + + GetNumberOfConsoleInputEvents = winexternal( + 'GetNumberOfConsoleInputEvents', [HANDLE, LPDWORD], BOOL) From pypy.commits at gmail.com Mon Jun 3 06:39:10 2019 From: pypy.commits at gmail.com (fijal) Date: Mon, 03 Jun 2019 03:39:10 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: New branch to play with rewriting copystrcontent Message-ID: <5cf4f8ce.1c69fb81.2401e.b02a@mx.google.com> Author: fijal Branch: copystrcontents-in-rewrite Changeset: r96728:d43e7d5dd376 Date: 2019-06-03 12:38 +0200 http://bitbucket.org/pypy/pypy/changeset/d43e7d5dd376/ Log: New branch to play with rewriting copystrcontent From pypy.commits at gmail.com Mon Jun 3 14:11:20 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 03 Jun 2019 11:11:20 -0700 (PDT) Subject: [pypy-commit] cffi default: Simplify the implementation, cut down the explanations, and allow Message-ID: <5cf562c8.1c69fb81.809f3.3443@mx.google.com> Author: Armin Rigo Branch: Changeset: r3276:5aa806ec6d00 Date: 2019-06-03 20:11 +0200 http://bitbucket.org/cffi/cffi/changeset/5aa806ec6d00/ Log: Simplify the implementation, cut down the explanations, and allow accessing multiple items instead of just one in from_buffer("type *"), trading off security for the realisation that it makes sense to access multiple items in some cases (unlike after ffi.new()). diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -248,8 +248,7 @@ Py_TYPE(ob) == &CDataFromBuf_Type || \ Py_TYPE(ob) == &CDataGCP_Type) #define CDataOwn_Check(ob) (Py_TYPE(ob) == &CDataOwning_Type || \ - Py_TYPE(ob) == &CDataOwningGC_Type || \ - Py_TYPE(ob) == &CDataFromBuf_Type) + Py_TYPE(ob) == &CDataOwningGC_Type) typedef union { unsigned char m_char; @@ -280,8 +279,7 @@ typedef struct { CDataObject head; - PyObject *structobj; /* for ffi.new_handle(), ffi.new("struct *"), - or ffi.from_buffer("struct *") */ + PyObject *structobj; /* for ffi.new_handle() or ffi.new("struct *") */ } CDataObject_own_structptr; typedef struct { @@ -1857,7 +1855,7 @@ assert(!(cd->c_type->ct_flags & (CT_IS_VOID_PTR | CT_FUNCTIONPTR))); if (cd->c_type->ct_flags & CT_IS_PTR_TO_OWNED) { - /* for ffi.new("struct *") or ffi.from_buffer("struct *") */ + /* for ffi.new("struct *") */ Py_DECREF(((CDataObject_own_structptr *)cd)->structobj); } #if defined(CFFI_MEM_DEBUG) || defined(CFFI_MEM_LEAK) @@ -2164,15 +2162,7 @@ static PyObject *cdataowning_repr(CDataObject *cd) { - Py_ssize_t size; - - if (cd->c_type->ct_flags & CT_IS_PTR_TO_OWNED) { - PyObject *x = ((CDataObject_own_structptr *)cd)->structobj; - if (Py_TYPE(x) == &CDataFromBuf_Type) - return _frombuf_repr((CDataObject *)x, cd->c_type->ct_name); - } - - size = _cdata_var_byte_size(cd); + Py_ssize_t size = _cdata_var_byte_size(cd); if (size < 0) { if (cd->c_type->ct_flags & CT_POINTER) size = cd->c_type->ct_itemdescr->ct_size; @@ -3214,7 +3204,7 @@ CTypeDescrObject *ct = ((CDataObject *)cd)->c_type; if (Py_TYPE(cd) == &CDataOwning_Type) { if ((ct->ct_flags & (CT_POINTER | CT_ARRAY)) != 0) /* ffi.new() */ - return 0; /* or ffi.from_buffer() */ + return 0; } else if (Py_TYPE(cd) == &CDataFromBuf_Type) { return 1; /* ffi.from_buffer() */ @@ -3257,11 +3247,6 @@ ffi.new_allocator()("struct-or-union *") */ cdatagcp_finalize((CDataObject_gcp *)x); } - else if (Py_TYPE(x) == &CDataFromBuf_Type) { - /* special case for ffi.from_buffer("struct *") */ - view = ((CDataObject_frombuf *)x)->bufferview; - PyBuffer_Release(view); - } } break; @@ -3497,7 +3482,7 @@ 0, /* inherited */ /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ - &CDataOwning_Type, /* tp_base */ + &CData_Type, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -6952,13 +6937,12 @@ return 0; } -static PyObject *direct_from_buffer(CTypeDescrObject *ct1, PyObject *x, +static PyObject *direct_from_buffer(CTypeDescrObject *ct, PyObject *x, int require_writable) { CDataObject *cd; - CTypeDescrObject *ct = ct1; Py_buffer *view; - Py_ssize_t arraylength, minimumlength = -1; + Py_ssize_t arraylength, minimumlength = 0; if (!(ct->ct_flags & (CT_ARRAY | CT_POINTER))) { PyErr_Format(PyExc_TypeError, @@ -6987,25 +6971,7 @@ if (ct->ct_flags & CT_POINTER) { - /* from_buffer("T*"): return a cdata pointer. Complain if the - size of the buffer is known to be smaller than sizeof(T), - i.e. never complain if sizeof(T) is not known at all. */ - CTypeDescrObject *ctitem = ct->ct_itemdescr; - minimumlength = ctitem->ct_size; /* maybe -1 */ - arraylength = -1; - - if (ct->ct_flags & CT_IS_PTR_TO_OWNED) { - ct = ctitem; - if (force_lazy_struct(ct) < 0) /* for CT_WITH_VAR_ARRAY */ - return NULL; - - if (ct->ct_flags & CT_WITH_VAR_ARRAY) { - assert(ct1->ct_flags & CT_IS_PTR_TO_OWNED); - arraylength = view->len; - if (ct->ct_size > 1) - arraylength = (arraylength / ct->ct_size) * ct->ct_size; - } - } + arraylength = view->len; /* number of bytes, not used so far */ } else { /* ct->ct_flags & CT_ARRAY */ @@ -7058,21 +7024,6 @@ ((CDataObject_frombuf *)cd)->length = arraylength; ((CDataObject_frombuf *)cd)->bufferview = view; PyObject_GC_Track(cd); - - if (ct1->ct_flags & CT_IS_PTR_TO_OWNED) /* ptr-to-struct/union */ - { - CDataObject *cds = cd; - cd = allocate_owning_object(sizeof(CDataObject_own_structptr), ct1, - /*dont_clear=*/1); - if (cd == NULL) { - Py_DECREF(cds); - goto error2; - } - /* store the only reference to cds into cd */ - ((CDataObject_own_structptr *)cd)->structobj = (PyObject *)cds; - - cd->c_data = cds->c_data; - } return (PyObject *)cd; error2: diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3864,12 +3864,13 @@ assert p2 == p1 assert typeof(p2) is BIntP assert p2[0] == lst[0] - with pytest.raises(IndexError): - p2[1] - with pytest.raises(IndexError): - p2[-1] - with pytest.raises(ValueError): - from_buffer(BIntP, b"abc") # not enough data + assert p2[1] == lst[1] + assert p2[2] == lst[2] + # hopefully does not crash, but doesn't raise an exception: + p2[3] + p2[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BIntP, b"") # BIntA2 = new_array_type(BIntP, 2) p2 = from_buffer(BIntA2, bytestring) # int[2] @@ -3911,26 +3912,18 @@ assert p2.a2 == lst2[1] assert p2[0].a1 == lst2[0] assert p2[0].a2 == lst2[1] - with pytest.raises(IndexError): - p2[1] - with pytest.raises(IndexError): - p2[-1] - with pytest.raises(ValueError): - from_buffer(BStructP, b"1234567") # not enough data + assert p2[1].a1 == lst2[2] + assert p2[1].a2 == lst2[3] + # does not crash: + p2[2] + p2[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BStructP, b"") + from_buffer(BStructP, b"1234567") # release(p1) assert repr(p1) == "" # - # keepalive behaviour similar to newp() - plst = [] - for i in range(100): - plst.append(from_buffer(BStructP, buffer(newp(BStructP, [i, -i])))[0]) - if i % 10 == 5: - import gc; gc.collect() - for i in range(len(plst)): - assert plst[i].a1 == i - assert plst[i].a2 == -i - # BEmptyStruct = new_struct_type("empty") complete_struct_or_union(BEmptyStruct, [], Ellipsis, 0) assert sizeof(BEmptyStruct) == 0 @@ -3955,25 +3948,24 @@ assert pv.a1 == lst[0] assert pv.va[0] == lst[1] assert pv.va[1] == lst[2] - assert sizeof(pv[0]) == 3 * size_of_int() - assert len(pv.va) == 2 - with pytest.raises(IndexError): - pv.va[2] - with pytest.raises(IndexError): - pv.va[-1] - with pytest.raises(ValueError): - from_buffer(BVarStructP, b"123") # not enough data + assert sizeof(pv[0]) == 1 * size_of_int() + with pytest.raises(TypeError): + len(pv.va) + # hopefully does not crash, but doesn't raise an exception: + pv.va[2] + pv.va[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BVarStructP, b"") assert repr(pv) == "" - assert repr(pv[0]) == "" + assert repr(pv[0]).startswith("" - assert repr(pv[0]) == "" + assert repr(pv[0]).startswith("" - assert repr(pv[0]) == "" + with pytest.raises(ValueError): + release(pv[0]) def test_memmove(): Short = new_primitive_type("short") diff --git a/doc/source/ref.rst b/doc/source/ref.rst --- a/doc/source/ref.rst +++ b/doc/source/ref.rst @@ -248,15 +248,11 @@ keeps the underlying Python object alive and locked. (In addition, ``ffi.from_buffer("int[]", x)`` gives better array bound checking.) - *New in version 1.13:* ``cdecl`` can be a pointer type. Then - ``from_buffer()`` raises a ValueError only if the type pointed to is of - known size and the buffer is smaller than that; but note that the result - ``p`` can only be indexed with ``p[0]`` even if the buffer is large enough - to contain several copies of the type. If the type is a struct or union, - this allows you to write ``p.field`` instead of the required ``p[0].field`` - if you had called ``from_buffer("struct s[1]")``; and also, in this case - either ``p`` or ``p[0]`` can be used to keep the buffer alive, similarly - to ``ffi.new()``. + *New in version 1.13:* ``cdecl`` can be a pointer type. If it points + to a struct or union, you can, as usual, write ``p.field`` instead of + ``p[0].field``. You can also access ``p[n]``; note that CFFI does not + perform any bounds checking in this case. Note also that ``p[0]`` cannot + be used to keep the buffer alive (unlike what occurs with ``ffi.new()``). * if ``require_writable`` is set to True, the function fails if the buffer obtained from ``python_buffer`` is read-only (e.g. if ``python_buffer`` is diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -7,13 +7,9 @@ ===== * ``ffi.from_buffer("type *", ..)`` is now supported, in addition to - ``"type[]"``. This is useful for buffers that contain only one ``type``. - (The ``type`` can also be a structure with an open-ended array as the last - item.) You can then write ``p.field`` to access the items, instead of - ``p[0].field``. The expression ``p[0]`` by itself behaves like it does - for ``p = ffi.new("struct-or-union *")``: you get a cdata object whose - type is the structure or union, but which keeps alive the original buffer - object even if ``p`` goes out of scope. + ``"type[]"``. You can then write ``p.field`` to access the items, instead + of ``p[0].field``. Be careful that no bounds checking is performed, so + ``p[n]`` might access data out of bounds. v1.12.3 From pypy.commits at gmail.com Mon Jun 3 14:34:29 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 03 Jun 2019 11:34:29 -0700 (PDT) Subject: [pypy-commit] cffi default: on py.py, bytearray buffers are not emulated well enough Message-ID: <5cf56835.1c69fb81.b49d6.5a6a@mx.google.com> Author: Armin Rigo Branch: Changeset: r3277:ff1afa334366 Date: 2019-06-03 20:34 +0200 http://bitbucket.org/cffi/cffi/changeset/ff1afa334366/ Log: on py.py, bytearray buffers are not emulated well enough diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -3861,7 +3861,8 @@ py.test.raises(TypeError, from_buffer, BInt, bytestring) # p2 = from_buffer(BIntP, bytestring) # int * - assert p2 == p1 + assert p2 == p1 or 'PY_DOT_PY' in globals() + # note: on py.py ^^^, bytearray buffers are not emulated well enough assert typeof(p2) is BIntP assert p2[0] == lst[0] assert p2[1] == lst[1] @@ -3882,7 +3883,7 @@ p2[2] with pytest.raises(IndexError): p2[-1] - assert p2 == p1 + assert p2 == p1 or 'PY_DOT_PY' in globals() # BIntA4 = new_array_type(BIntP, 4) # int[4]: too big py.test.raises(ValueError, from_buffer, BIntA4, bytestring) @@ -3906,7 +3907,7 @@ assert repr(p1) == "" # p2 = from_buffer(BStructP, bytestring2) # 'struct *' - assert p2 == p1 + assert p2 == p1 or 'PY_DOT_PY' in globals() assert typeof(p2) is BStructP assert p2.a1 == lst2[0] assert p2.a2 == lst2[1] @@ -3936,7 +3937,8 @@ p1 = from_buffer(BEmptyStructA5, bytestring) # struct empty[5] assert typeof(p1) is BEmptyStructA5 assert len(p1) == 5 - assert cast(BIntP, p1) == from_buffer(BIntA, bytestring) + assert (cast(BIntP, p1) == from_buffer(BIntA, bytestring) + or 'PY_DOT_PY' in globals()) # BVarStruct = new_struct_type("varfoo") BVarStructP = new_pointer_type(BVarStruct) From pypy.commits at gmail.com Mon Jun 3 14:41:46 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 03 Jun 2019 11:41:46 -0700 (PDT) Subject: [pypy-commit] cffi default: bump version number to 1.13 Message-ID: <5cf569ea.1c69fb81.5e0ac.ceac@mx.google.com> Author: Armin Rigo Branch: Changeset: r3278:90d267a96234 Date: 2019-06-03 20:41 +0200 http://bitbucket.org/cffi/cffi/changeset/90d267a96234/ Log: bump version number to 1.13 diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" -#define CFFI_VERSION "1.12.3" +#define CFFI_VERSION "1.13.0" #ifdef MS_WIN32 #include diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -14,7 +14,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.12.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.13.0", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/cffi/__init__.py b/cffi/__init__.py --- a/cffi/__init__.py +++ b/cffi/__init__.py @@ -5,8 +5,8 @@ from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.12.3" -__version_info__ = (1, 12, 3) +__version__ = "1.13.0" +__version_info__ = (1, 13, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/cffi/_embedding.h b/cffi/_embedding.h --- a/cffi/_embedding.h +++ b/cffi/_embedding.h @@ -224,7 +224,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.12.3" + "\ncompiled with cffi version: 1.13.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/doc/source/conf.py b/doc/source/conf.py --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '1.12' +version = '1.13' # The full version, including alpha/beta/rc tags. -release = '1.12.3' +release = '1.13.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/installation.rst b/doc/source/installation.rst --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -52,13 +52,13 @@ * https://pypi.python.org/pypi/cffi -* Checksums of the "source" package version 1.12.3: +* Checksums of the "source" package version 1.13.0: - - MD5: 35ad1f9e1003cac9404c1493eb10d7f5 + - MD5: ... - - SHA: ccc49cf31bc3f4248f45b9ec83685e4e8090a9fa + - SHA: ... - - SHA256: 041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774 + - SHA256: ... * Or grab the most current version from the `Bitbucket page`_: ``hg clone https://bitbucket.org/cffi/cffi`` diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -198,7 +198,7 @@ `Mailing list `_ """, - version='1.12.3', + version='1.13.0', packages=['cffi'] if cpython else [], package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', '_embedding.h', '_cffi_errors.h']} From pypy.commits at gmail.com Mon Jun 3 14:50:23 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 03 Jun 2019 11:50:23 -0700 (PDT) Subject: [pypy-commit] pypy default: ffi.from_buffer("type *") Message-ID: <5cf56bef.1c69fb81.4eb1e.7853@mx.google.com> Author: Armin Rigo Branch: Changeset: r96729:d9c0166080e4 Date: 2019-06-03 20:39 +0200 http://bitbucket.org/pypy/pypy/changeset/d9c0166080e4/ Log: ffi.from_buffer("type *") diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -677,11 +677,14 @@ return self.length def _repr_extra(self): - if self.w_keepalive is not None: - name = self.space.type(self.w_keepalive).name + from pypy.module._cffi_backend import ctypearray + if self.w_keepalive is None: + return "buffer RELEASED" + obj_tp_name = self.space.type(self.w_keepalive).name + if isinstance(self.ctype, ctypearray.W_CTypeArray): + return "buffer len %d from '%s' object" % (self.length, obj_tp_name) else: - name = "(released)" - return "buffer len %d from '%s' object" % (self.length, name) + return "buffer from '%s' object" % (obj_tp_name,) def enter_exit(self, exit_now): # for now, limited effect on PyPy diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -112,9 +112,10 @@ @unwrap_spec(w_ctype=ctypeobj.W_CType, require_writable=int) def from_buffer(space, w_ctype, w_x, require_writable=0): - from pypy.module._cffi_backend import ctypearray - if not isinstance(w_ctype, ctypearray.W_CTypeArray): - raise oefmt(space.w_TypeError, "expected an array ctype, got '%s'", + from pypy.module._cffi_backend import ctypeptr, ctypearray + if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray): + raise oefmt(space.w_TypeError, + "expected a poiunter or array ctype, got '%s'", w_ctype.name) if space.isinstance_w(w_x, space.w_unicode): raise oefmt(space.w_TypeError, @@ -135,33 +136,36 @@ "raw address on PyPy", w_x) # buffersize = buf.getlength() - arraylength = w_ctype.length - if arraylength >= 0: - # it's an array with a fixed length; make sure that the - # buffer contains enough bytes. - if buffersize < w_ctype.size: - raise oefmt(space.w_ValueError, - "buffer is too small (%d bytes) for '%s' (%d bytes)", - buffersize, w_ctype.name, w_ctype.size) + if not isinstance(w_ctype, ctypearray.W_CTypeArray): + arraylength = buffersize # number of bytes, not used so far else: - # it's an open 'array[]' - itemsize = w_ctype.ctitem.size - if itemsize == 1: - # fast path, performance only - arraylength = buffersize - elif itemsize > 0: - # give it as many items as fit the buffer. Ignore a - # partial last element. - arraylength = buffersize / itemsize + arraylength = w_ctype.length + if arraylength >= 0: + # it's an array with a fixed length; make sure that the + # buffer contains enough bytes. + if buffersize < w_ctype.size: + raise oefmt(space.w_ValueError, + "buffer is too small (%d bytes) for '%s' (%d bytes)", + buffersize, w_ctype.name, w_ctype.size) else: - # it's an array 'empty[]'. Unsupported obscure case: - # the problem is that setting the length of the result - # to anything large (like SSIZE_T_MAX) is dangerous, - # because if someone tries to loop over it, it will - # turn effectively into an infinite loop. - raise oefmt(space.w_ZeroDivisionError, - "from_buffer('%s', ..): the actual length of the array " - "cannot be computed", w_ctype.name) + # it's an open 'array[]' + itemsize = w_ctype.ctitem.size + if itemsize == 1: + # fast path, performance only + arraylength = buffersize + elif itemsize > 0: + # give it as many items as fit the buffer. Ignore a + # partial last element. + arraylength = buffersize / itemsize + else: + # it's an array 'empty[]'. Unsupported obscure case: + # the problem is that setting the length of the result + # to anything large (like SSIZE_T_MAX) is dangerous, + # because if someone tries to loop over it, it will + # turn effectively into an infinite loop. + raise oefmt(space.w_ZeroDivisionError, + "from_buffer('%s', ..): the actual length of the array " + "cannot be computed", w_ctype.name) # return cdataobj.W_CDataFromBuffer(space, _cdata, arraylength, w_ctype, buf, w_x) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3830,7 +3830,9 @@ BIntP = new_pointer_type(BInt) BIntA = new_array_type(BIntP, None) lst = [-12345678, 87654321, 489148] - bytestring = buffer(newp(BIntA, lst))[:] + b'XYZ' + bytestring = bytearray(buffer(newp(BIntA, lst))[:] + b'XYZ') + lst2 = lst + [42, -999999999] + bytestring2 = bytearray(buffer(newp(BIntA, lst2))[:] + b'XYZ') # p1 = from_buffer(BIntA, bytestring) # int[] assert typeof(p1) is BIntA @@ -3844,7 +3846,19 @@ p1[-1] # py.test.raises(TypeError, from_buffer, BInt, bytestring) - py.test.raises(TypeError, from_buffer, BIntP, bytestring) + # + p2 = from_buffer(BIntP, bytestring) # int * + assert p2 == p1 or 'PY_DOT_PY' in globals() + # note: on py.py ^^^, bytearray buffers are not emulated well enough + assert typeof(p2) is BIntP + assert p2[0] == lst[0] + assert p2[1] == lst[1] + assert p2[2] == lst[2] + # hopefully does not crash, but doesn't raise an exception: + p2[3] + p2[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BIntP, b"") # BIntA2 = new_array_type(BIntP, 2) p2 = from_buffer(BIntA2, bytestring) # int[2] @@ -3856,7 +3870,7 @@ p2[2] with pytest.raises(IndexError): p2[-1] - assert p2 == p1 + assert p2 == p1 or 'PY_DOT_PY' in globals() # BIntA4 = new_array_type(BIntP, 4) # int[4]: too big py.test.raises(ValueError, from_buffer, BIntA4, bytestring) @@ -3866,13 +3880,37 @@ ('a2', BInt, -1)]) BStructP = new_pointer_type(BStruct) BStructA = new_array_type(BStructP, None) - p1 = from_buffer(BStructA, bytestring) # struct[] - assert len(p1) == 1 + p1 = from_buffer(BStructA, bytestring2) # struct[] + assert len(p1) == 2 assert typeof(p1) is BStructA - assert p1[0].a1 == lst[0] - assert p1[0].a2 == lst[1] + assert p1[0].a1 == lst2[0] + assert p1[0].a2 == lst2[1] + assert p1[1].a1 == lst2[2] + assert p1[1].a2 == lst2[3] with pytest.raises(IndexError): - p1[1] + p1[2] + with pytest.raises(IndexError): + p1[-1] + assert repr(p1) == "" + # + p2 = from_buffer(BStructP, bytestring2) # 'struct *' + assert p2 == p1 or 'PY_DOT_PY' in globals() + assert typeof(p2) is BStructP + assert p2.a1 == lst2[0] + assert p2.a2 == lst2[1] + assert p2[0].a1 == lst2[0] + assert p2[0].a2 == lst2[1] + assert p2[1].a1 == lst2[2] + assert p2[1].a2 == lst2[3] + # does not crash: + p2[2] + p2[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BStructP, b"") + from_buffer(BStructP, b"1234567") + # + release(p1) + assert repr(p1) == "" # BEmptyStruct = new_struct_type("empty") complete_struct_or_union(BEmptyStruct, [], Ellipsis, 0) @@ -3886,7 +3924,37 @@ p1 = from_buffer(BEmptyStructA5, bytestring) # struct empty[5] assert typeof(p1) is BEmptyStructA5 assert len(p1) == 5 - assert cast(BIntP, p1) == from_buffer(BIntA, bytestring) + assert (cast(BIntP, p1) == from_buffer(BIntA, bytestring) + or 'PY_DOT_PY' in globals()) + # + BVarStruct = new_struct_type("varfoo") + BVarStructP = new_pointer_type(BVarStruct) + complete_struct_or_union(BVarStruct, [('a1', BInt, -1), + ('va', BIntA, -1)]) + with pytest.raises(TypeError): + from_buffer(BVarStruct, bytestring) + pv = from_buffer(BVarStructP, bytestring) # varfoo * + assert pv.a1 == lst[0] + assert pv.va[0] == lst[1] + assert pv.va[1] == lst[2] + assert sizeof(pv[0]) == 1 * size_of_int() + with pytest.raises(TypeError): + len(pv.va) + # hopefully does not crash, but doesn't raise an exception: + pv.va[2] + pv.va[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BVarStructP, b"") + assert repr(pv) == "" + assert repr(pv[0]).startswith("" + assert repr(pv[0]).startswith("" release(p) assert p[2] == b"z" # true so far, but might change to raise RuntimeError + assert repr(p) == "" release(p) # no effect def test_explicit_release_from_buffer_contextmgr(): @@ -4325,6 +4395,7 @@ with p: assert p[2] == b"z" assert p[2] == b"z" # true so far, but might change to raise RuntimeError + assert repr(p) == "" release(p) # no effect def test_explicit_release_bytearray_on_cpython(): From pypy.commits at gmail.com Mon Jun 3 14:50:25 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 03 Jun 2019 11:50:25 -0700 (PDT) Subject: [pypy-commit] pypy default: update to cffi/90d267a96234 Message-ID: <5cf56bf1.1c69fb81.944c7.a31b@mx.google.com> Author: Armin Rigo Branch: Changeset: r96730:cbe1542365b6 Date: 2019-06-03 20:47 +0200 http://bitbucket.org/pypy/pypy/changeset/cbe1542365b6/ Log: update to cffi/90d267a96234 diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py --- a/extra_tests/cffi_tests/cffi0/test_parsing.py +++ b/extra_tests/cffi_tests/cffi0/test_parsing.py @@ -410,7 +410,17 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; + enum Enum { + POS = +1, + TWO = 2, + NIL = 0, + NEG = -1, + ADDSUB = (POS+TWO)-1, + DIVMULINT = (3 * 3) / 2, + SHIFT = (1 << 3) >> 1, + BINOPS = (0x7 & 0x1) | 0x8, + XOR = 0xf ^ 0xa + }; """) needs_dlopen_none() C = ffi.dlopen(None) @@ -418,7 +428,11 @@ assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 - assert C.OP == 2 + assert C.ADDSUB == 2 + assert C.DIVMULINT == 4 + assert C.SHIFT == 4 + assert C.BINOPS == 0b1001 + assert C.XOR == 0b0101 def test_stdcall(): ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py --- a/extra_tests/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2535,3 +2535,29 @@ x.p = p x.cyclic = x del p, x + +def test_arithmetic_in_cdef(): + for a in [0, 11, 15]: + ffi = FFI() + ffi.cdef(""" + enum FOO { + DIVNN = ((-?) / (-3)), + DIVNP = ((-?) / (+3)), + DIVPN = ((+?) / (-3)), + MODNN = ((-?) % (-3)), + MODNP = ((-?) % (+3)), + MODPN = ((+?) % (-3)), + }; + """.replace('?', str(a))) + lib = ffi.verify(""" + enum FOO { + DIVNN = ((-?) / (-3)), + DIVNP = ((-?) / (+3)), + DIVPN = ((+?) / (-3)), + MODNN = ((-?) % (-3)), + MODNP = ((-?) % (+3)), + MODPN = ((+?) % (-3)), + }; + """.replace('?', str(a))) + # the verify() crashes if the values in the enum are different from + # the values we computed ourselves from the cdef() diff --git a/extra_tests/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/extra_tests/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -2,11 +2,13 @@ import py, os, sys, shutil import subprocess from extra_tests.cffi_tests.udir import udir +import pytest if sys.platform == 'win32': - py.test.skip('snippets do not run on win32') + pytestmark = pytest.mark.skip('snippets do not run on win32') if sys.version_info < (2, 7): - py.test.skip('fails e.g. on a Debian/Ubuntu which patches virtualenv' + pytestmark = pytest.mark.skip( + 'fails e.g. on a Debian/Ubuntu which patches virtualenv' ' in a non-2.6-friendly way') def create_venv(name): diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -2339,3 +2339,24 @@ typedef int foo_t; struct foo_s { void (*x)(foo_t); }; """) py.test.raises(TypeError, ffi.new, "struct foo_s *") + +def test_from_buffer_struct(): + ffi = FFI() + ffi.cdef("""struct foo_s { int a, b; };""") + lib = verify(ffi, "test_from_buffer_struct_p", """ + struct foo_s { int a, b; }; + """) + p = ffi.new("struct foo_s *", [-219239, 58974983]) + q = ffi.from_buffer("struct foo_s[]", ffi.buffer(p)) + assert ffi.typeof(q) == ffi.typeof("struct foo_s[]") + assert len(q) == 1 + assert q[0].a == p.a + assert q[0].b == p.b + assert q == p + q = ffi.from_buffer("struct foo_s *", ffi.buffer(p)) + assert ffi.typeof(q) == ffi.typeof("struct foo_s *") + assert q.a == p.a + assert q.b == p.b + assert q[0].a == p.a + assert q[0].b == p.b + assert q == p diff --git a/extra_tests/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py --- a/extra_tests/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -64,8 +64,8 @@ output = popen.stdout.read() err = popen.wait() if err: - raise OSError("popen failed with exit code %r: %r" % ( - err, args)) + raise OSError(("popen failed with exit code %r: %r\n\n%s" % ( + err, args, output)).rstrip()) print(output.rstrip()) return output diff --git a/extra_tests/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py --- a/extra_tests/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -3,8 +3,8 @@ from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': - import py - py.test.skip("written with POSIX functions") + import pytest + pytestmark = pytest.mark.skip("written with POSIX functions") class TestPerformance(EmbeddingTests): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.12.3 +Version: 1.13.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -5,8 +5,8 @@ from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.12.3" -__version_info__ = (1, 12, 3) +__version__ = "1.13.0" +__version_info__ = (1, 13, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -145,6 +145,7 @@ int result; PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; + PyObject *builtins; state = PyGILState_Ensure(); @@ -169,7 +170,7 @@ global_dict = PyDict_New(); if (global_dict == NULL) goto error; - PyObject *builtins = PyEval_GetBuiltins(); + builtins = PyEval_GetBuiltins(); if (builtins == NULL) goto error; if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) @@ -223,7 +224,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.12.3" + "\ncompiled with cffi version: 1.13.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -858,19 +858,39 @@ "the actual array length in this context" % exprnode.coord.line) # - if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and - exprnode.op == '+'): - return (self._parse_constant(exprnode.left) + - self._parse_constant(exprnode.right)) - # - if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and - exprnode.op == '-'): - return (self._parse_constant(exprnode.left) - - self._parse_constant(exprnode.right)) + if isinstance(exprnode, pycparser.c_ast.BinaryOp): + left = self._parse_constant(exprnode.left) + right = self._parse_constant(exprnode.right) + if exprnode.op == '+': + return left + right + elif exprnode.op == '-': + return left - right + elif exprnode.op == '*': + return left * right + elif exprnode.op == '/': + return self._c_div(left, right) + elif exprnode.op == '%': + return left - self._c_div(left, right) * right + elif exprnode.op == '<<': + return left << right + elif exprnode.op == '>>': + return left >> right + elif exprnode.op == '&': + return left & right + elif exprnode.op == '|': + return left | right + elif exprnode.op == '^': + return left ^ right # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) + def _c_div(self, a, b): + result = a // b + if ((a < 0) ^ (b < 0)) and (a % b) != 0: + result += 1 + return result + def _build_enum_type(self, explicit_name, decls): if decls is not None: partial = False diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.12.3" +VERSION = "1.13.0" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.12.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.13.0", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): From pypy.commits at gmail.com Tue Jun 4 06:48:59 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 04 Jun 2019 03:48:59 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: an attempt to rewrite copystr/copyunicode content Message-ID: <5cf64c9b.1c69fb81.4ec10.b976@mx.google.com> Author: fijal Branch: copystrcontents-in-rewrite Changeset: r96731:825a9d694c31 Date: 2019-06-04 12:48 +0200 http://bitbucket.org/pypy/pypy/changeset/825a9d694c31/ Log: an attempt to rewrite copystr/copyunicode content diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -14,6 +14,7 @@ from rpython.jit.metainterp.support import ptr2int from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.symbolic import WORD +from rpython.jit.backend.llsupport.memcpy import memcpy_fn from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr, FieldDescr from rpython.jit.backend.llsupport.descr import GcCache, get_field_descr from rpython.jit.backend.llsupport.descr import get_array_descr @@ -36,6 +37,11 @@ self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, 'typeptr') self._generated_functions = [] + self.memcpy_fn = memcpy_fn + self.memcpy_descr = get_call_descr(self, + [lltype.Signed, lltype.Signed, lltype.Signed], lltype.Void, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, + can_collect=False)) def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -34,6 +34,10 @@ - Add COND_CALLs to the write barrier before SETFIELD_GC and SETARRAYITEM_GC operations. + - Rewrites copystrcontent to a call to memcopy + + - XXX does more than that, please write it down + '_write_barrier_applied' contains a dictionary of variable -> None. If a variable is in the dictionary, next setfields can be called without a write barrier. The idea is that an object that was freshly allocated @@ -335,6 +339,10 @@ self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: self.emit_label() + # ---- change COPY{STR|UNICODE}CONTENT into a call ------ + if op.opnum == rop.COPYSTRCONTENT or op.opnum == rop.COPYUNICODECONTENT: + self.rewrite_copy_str_content(op) + continue # ---------- write barriers ---------- if self.gc_ll_descr.write_barrier_descr is not None: if op.getopnum() == rop.SETFIELD_GC: @@ -953,6 +961,46 @@ self.gcrefs_output_list.append(gcref) return index + def rewrite_copy_str_content(self, op): + funcaddr = llmemory.cast_ptr_to_adr(self.gc_ll_descr.memcpy_fn) + memcpy_fn = self.cpu.cast_adr_to_int(funcaddr) + memcpy_descr = self.gc_ll_descr.memcpy_descr + if op.getopnum() == rop.COPYSTRCONTENT: + basesize = self.gc_ll_descr.str_descr.basesize + # because we have one extra item after alloc, the actual address + # of string start is 1 lower, from extra_item_after_malloc + base = ConstInt(basesize - 1) + itemsize = self.gc_ll_descr.str_descr.itemsize + assert itemsize == 1 + itemscale = ConstInt(0) + else: + base = ConstInt(self.gc_ll_descr.unicode_descr.basesize) + itemsize = self.gc_ll_descr.unicode_descr.itemsize + if itemsize == 2: + itemscale = ConstInt(1) + elif itemsize == 4: + itemscale = ConstInt(2) + else: + assert False, "unknown size of unicode" + i1 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS, + [op.getarg(0), op.getarg(2), base, itemscale]) + i2 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS, + [op.getarg(1), op.getarg(3), base, itemscale]) + self.emit_op(i1) + self.emit_op(i2) + if op.getopnum() == rop.COPYSTRCONTENT: + arg = op.getarg(4) + else: + # do some basic constant folding + if isinstance(op.getarg(4), ConstInt): + arg = ConstInt(op.getarg(4).getint() * itemsize) + else: + arg = ResOperation(rop.INT_MUL, [op.getarg(4), ConstInt(itemsize)]) + self.emit_op(arg) + self.emit_op(ResOperation(rop.CALL_N, + [ConstInt(memcpy_fn), i2, i1, arg], descr=memcpy_descr)) + + def remove_constptr(self, c): """Remove all ConstPtrs, and replace them with load_from_gc_table. """ diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -147,6 +147,8 @@ unicodelendescr = unicodedescr.lendescr strhashdescr = self.gc_ll_descr.str_hash_descr unicodehashdescr = self.gc_ll_descr.unicode_hash_descr + memcpy_fn = self.gc_ll_descr.memcpy_fn + memcpy_descr = self.gc_ll_descr.memcpy_descr casmdescr = JitCellToken() clt = FakeLoopToken() @@ -169,6 +171,7 @@ signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr casmdescr.compiled_loop_token = clt + # guarddescr = AbstractFailDescr() # @@ -237,6 +240,9 @@ self._cache[key] = r return r + def cast_adr_to_int(self, adr): + return rffi.cast(lltype.Signed, adr) + class TestBoehm(RewriteTests): def setup_method(self, meth): class FakeCPU(BaseFakeCPU): @@ -1436,3 +1442,14 @@ jump() """) assert len(self.gcrefs) == 2 + + def test_rewrite_copystrcontents(self): + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copystrcontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2 = load_effective_address(p0, i0, %(strdescr.basesize)s, %(strdescr.itemsize)s) + i3 = load_effective_address(p1, i1, %(strdescr.basesize)s, %(strdescr.itemsize)s) + call_n(ConstClass(memcpy_fn), i2, i3, i_len, descr=memcpy_descr) + """) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1222,6 +1222,17 @@ resloc = self.force_allocate_reg(op, [op.getarg(0)]) self.perform(op, [argloc], resloc) + def consider_load_effective_address(self, op): + p0 = op.getarg(0) + i0 = op.getarg(1) + ploc = self.make_sure_var_in_reg(p0, [i0]) + iloc = self.make_sure_var_in_reg(i0, [p0]) + res = self.rm.force_allocate_reg(op, [p0, i0]) + assert isinstance(op.getarg(2), ConstInt) + assert isinstance(op.getarg(3), ConstInt) + self.assembler.load_effective_addr(iloc, op.getarg(2).getint(), + op.getarg(3).getint(), res, ploc) + def consider_copystrcontent(self, op): self._consider_copystrcontent(op, is_unicode=False) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -441,6 +441,7 @@ rop.GC_STORE, rop.GC_STORE_INDEXED, rop.LOAD_FROM_GC_TABLE, + rop.LOAD_EFFECTIVE_ADDRESS, ): # list of opcodes never executed by pyjitpl continue if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST: diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1056,6 +1056,8 @@ # 'LOAD_FROM_GC_TABLE/1/r', # only emitted by rewrite.py # + 'LOAD_EFFECTIVE_ADDRESS/4/i', # only emitted by rewrite.py, shortcut for x86 + # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- # parameters GC_LOAD From pypy.commits at gmail.com Tue Jun 4 07:00:36 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 04 Jun 2019 04:00:36 -0700 (PDT) Subject: [pypy-commit] pypy default: win32, python2 doesn't like surrogates, skip this check Message-ID: <5cf64f54.1c69fb81.f7fe.8af4@mx.google.com> Author: Matti Picus Branch: Changeset: r96732:5604ceb9aa78 Date: 2019-05-30 16:51 +0300 http://bitbucket.org/pypy/pypy/changeset/5604ceb9aa78/ Log: win32, python2 doesn't like surrogates, skip this check diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1,5 +1,7 @@ """The builtin unicode implementation""" +import sys + from rpython.rlib.objectmodel import ( compute_hash, compute_unique_id, import_from_mixin, always_inline, enforceargs, newlist_hint, specialize, we_are_translated) @@ -41,7 +43,7 @@ self._utf8 = utf8str self._length = length self._index_storage = rutf8.null_storage() - if not we_are_translated(): + if not we_are_translated() and not sys.platform == 'win32': # utf8str must always be a valid utf8 string, except maybe with # explicit surrogate characters---which .decode('utf-8') doesn't # special-case in Python 2, which is exactly what we want here From pypy.commits at gmail.com Tue Jun 4 07:33:56 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 04 Jun 2019 04:33:56 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: fix test_rewrite Message-ID: <5cf65724.1c69fb81.af32e.d927@mx.google.com> Author: fijal Branch: copystrcontents-in-rewrite Changeset: r96733:cb6c60bbc714 Date: 2019-06-04 13:30 +0200 http://bitbucket.org/pypy/pypy/changeset/cb6c60bbc714/ Log: fix test_rewrite diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -142,6 +142,7 @@ raw_sfdescr = get_array_descr(self.gc_ll_descr, RAW_SF) # strdescr = self.gc_ll_descr.str_descr + str_basesize = self.gc_ll_descr.str_descr.basesize - 1 unicodedescr = self.gc_ll_descr.unicode_descr strlendescr = strdescr.lendescr unicodelendescr = unicodedescr.lendescr @@ -241,7 +242,7 @@ return r def cast_adr_to_int(self, adr): - return rffi.cast(lltype.Signed, adr) + return llmemory.AddressAsInt(adr) class TestBoehm(RewriteTests): def setup_method(self, meth): @@ -1449,7 +1450,7 @@ copystrcontent(p0, p1, i0, i1, i_len) """, """ [p0, p1, i0, i1, i_len] - i2 = load_effective_address(p0, i0, %(strdescr.basesize)s, %(strdescr.itemsize)s) - i3 = load_effective_address(p1, i1, %(strdescr.basesize)s, %(strdescr.itemsize)s) - call_n(ConstClass(memcpy_fn), i2, i3, i_len, descr=memcpy_descr) + i2 = load_effective_address(p0, i0, %(str_basesize)s, 0) + i3 = load_effective_address(p1, i1, %(str_basesize)s, 0) + call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) """) From pypy.commits at gmail.com Tue Jun 4 07:52:59 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Jun 2019 04:52:59 -0700 (PDT) Subject: [pypy-commit] pypy optimizeopt-cleanup: remove broken tests Message-ID: <5cf65b9b.1c69fb81.a94ee.a463@mx.google.com> Author: Ronan Lamy Branch: optimizeopt-cleanup Changeset: r96734:3d5f7507303f Date: 2019-05-28 01:33 +0100 http://bitbucket.org/pypy/pypy/changeset/3d5f7507303f/ Log: remove broken tests diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -7,7 +7,7 @@ TAGBOX, TAGVIRTUAL, tagged_list_eq, AbstractVirtualInfo, TAGCONST, NULLREF, ResumeDataDirectReader, TAGINT, REF, VirtualInfo, VStructInfo, VArrayInfoNotClear, VStrPlainInfo, VStrConcatInfo, VStrSliceInfo, - VUniPlainInfo, VUniConcatInfo, VUniSliceInfo, capture_resumedata, + VUniPlainInfo, VUniConcatInfo, VUniSliceInfo, ResumeDataLoopMemo, UNASSIGNEDVIRTUAL, INT, annlowlevel, PENDINGFIELDSP, TAG_CONST_OFFSET) from rpython.jit.metainterp.resumecode import ( @@ -56,7 +56,7 @@ def dump_storage(storage, liveboxes): "For profiling only." debug_start("jit-resume") - return # XXX refactor if needed + return # XXX refactor if needed if have_debug_prints(): debug_print('Log storage', compute_unique_id(storage)) frameinfo = storage.rd_frame_info_list @@ -515,8 +515,10 @@ def __eq__(self, other): return self.__dict__ == other.__dict__ + def __ne__(self, other): return self.__dict__ != other.__dict__ + def __repr__(self): return "" % (self.jitcode, self.pc, self._env) @@ -532,228 +534,6 @@ class options: failargs_limit = 100 -def test_rebuild_from_resumedata(): - py.test.skip("XXX rewrite") - b1, b2, b3 = [BoxInt(), InputArgRef(), BoxInt()] - c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] - storage = Storage() - fs = [FakeFrame("code0", 0, b1, c1, b2), - FakeFrame("code1", 3, b3, c2, b1), - FakeFrame("code2", 9, c3, b2)] - capture_resumedata(fs, None, [], storage) - memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(FakeOptimizer(), storage, storage, memo) - liveboxes = modifier.finish() - metainterp = MyMetaInterp() - - b1t, b2t, b3t = [BoxInt(), InputArgRef(), BoxInt()] - newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t) - - result = rebuild_from_resumedata(metainterp, storage, False) - assert result == (None, []) - fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), - FakeFrame("code1", 3, b3t, c2, b1t), - FakeFrame("code2", 9, c3, b2t)] - assert metainterp.framestack == fs2 - -def test_rebuild_from_resumedata_with_virtualizable(): - py.test.skip("XXX rewrite") - b1, b2, b3, b4 = [BoxInt(), InputArgRef(), BoxInt(), InputArgRef()] - c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] - storage = Storage() - fs = [FakeFrame("code0", 0, b1, c1, b2), - FakeFrame("code1", 3, b3, c2, b1), - FakeFrame("code2", 9, c3, b2)] - capture_resumedata(fs, [b4], [], storage) - memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(FakeOptimizer({}), storage, memo) - liveboxes = modifier.finish() - metainterp = MyMetaInterp() - - b1t, b2t, b3t, b4t = [BoxInt(), InputArgRef(), BoxInt(), InputArgRef()] - newboxes = _resume_remap(liveboxes, [b1, b2, b3, b4], b1t, b2t, b3t, b4t) - - result = rebuild_from_resumedata(metainterp, newboxes, storage, - True) - assert result == ([b4t], []) - fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), - FakeFrame("code1", 3, b3t, c2, b1t), - FakeFrame("code2", 9, c3, b2t)] - assert metainterp.framestack == fs2 - -def test_rebuild_from_resumedata_two_guards(): - py.test.skip("XXX rewrite") - b1, b2, b3, b4 = [BoxInt(), InputArgRef(), BoxInt(), BoxInt()] - c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] - storage = Storage() - fs = [FakeFrame("code0", 0, b1, c1, b2), - FakeFrame("code1", 3, b3, c2, b1), - FakeFrame("code2", 9, c3, b2)] - capture_resumedata(fs, None, [], storage) - storage2 = Storage() - fs = fs[:-1] + [FakeFrame("code2", 10, c3, b2, b4)] - capture_resumedata(fs, None, [], storage2) - - memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(FakeOptimizer({}), storage, memo) - liveboxes = modifier.finish() - - modifier = ResumeDataVirtualAdder(FakeOptimizer({}), storage2, memo) - liveboxes2 = modifier.finish() - - metainterp = MyMetaInterp() - - b1t, b2t, b3t, b4t = [BoxInt(), InputArgRef(), BoxInt(), BoxInt()] - newboxes = _resume_remap(liveboxes, [b1, b2, b3], b1t, b2t, b3t) - - result = rebuild_from_resumedata(metainterp, newboxes, storage, - False) - assert result == (None, []) - fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), - FakeFrame("code1", 3, b3t, c2, b1t), - FakeFrame("code2", 9, c3, b2t)] - assert metainterp.framestack == fs2 - - newboxes = _resume_remap(liveboxes2, [b1, b2, b3, b4], b1t, b2t, b3t, b4t) - - metainterp.framestack = [] - result = rebuild_from_resumedata(metainterp, newboxes, storage2, - False) - assert result == (None, []) - fs2 = fs2[:-1] + [FakeFrame("code2", 10, c3, b2t, b4t)] - assert metainterp.framestack == fs2 - - -class FakeOptimizer_VirtualValue(object): - class optimizer: - class cpu: - pass -fakeoptimizer = FakeOptimizer_VirtualValue() - -def virtual_value(keybox, value, next): - vv = VirtualValue( - fakeoptimizer, ConstInt(ptr2int(LLtypeMixin.node_vtable)), keybox) - if not isinstance(next, OptValue): - next = OptValue(next) - vv.setfield(LLtypeMixin.valuedescr, OptValue(value)) - vv.setfield(LLtypeMixin.nextdescr, next) - return vv - -def test_rebuild_from_resumedata_two_guards_w_virtuals(): - py.test.skip("XXX rewrite") - - b1, b2, b3, b4, b5 = [BoxInt(), InputArgRef(), BoxInt(), BoxInt(), BoxInt()] - c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), - LLtypeMixin.nodebox.constbox()] - storage = Storage() - fs = [FakeFrame("code0", 0, b1, c1, b2), - FakeFrame("code1", 3, b3, c2, b1), - FakeFrame("code2", 9, c3, b2)] - capture_resumedata(fs, None, [], storage) - storage2 = Storage() - fs = fs[:-1] + [FakeFrame("code2", 10, c3, b2, b4)] - capture_resumedata(fs, None, [], storage2) - - memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - values = {b2: virtual_value(b2, b5, c4)} - modifier = ResumeDataVirtualAdder(FakeOptimizer(values), storage, memo) - liveboxes = modifier.finish() - assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), - tag(0, TAGCONST)] - - b6 = InputArgRef() - v6 = virtual_value(b6, c2, None) - v6.setfield(LLtypeMixin.nextdescr, v6) - values = {b2: virtual_value(b2, b4, v6), b6: v6} - memo.clear_box_virtual_numbers() - modifier = ResumeDataVirtualAdder(FakeOptimizer(values), storage2, memo) - liveboxes2 = modifier.finish() - assert len(storage2.rd_virtuals) == 2 - assert storage2.rd_virtuals[0].fieldnums == [tag(len(liveboxes2)-1, TAGBOX), - tag(-1, TAGVIRTUAL)] - assert storage2.rd_virtuals[1].fieldnums == [tag(2, TAGINT), - tag(-1, TAGVIRTUAL)] - - # now on to resuming - metainterp = MyMetaInterp() - - b1t, b3t, b4t, b5t = [BoxInt(), BoxInt(), BoxInt(), BoxInt()] - newboxes = _resume_remap(liveboxes, [b1, b3, b5], b1t, b3t, b5t) - - result = rebuild_from_resumedata(metainterp, newboxes, storage, - False) - - b2t = metainterp.resboxes[0] - fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), - FakeFrame("code1", 3, b3t, c2, b1t), - FakeFrame("code2", 9, c3, b2t)] - assert metainterp.framestack == fs2 - - newboxes = _resume_remap(liveboxes2, [b1, b3, b4], b1t, b3t, b4t) - - metainterp = MyMetaInterp() - result = rebuild_from_resumedata(metainterp, newboxes, storage2, - False) - b2t = metainterp.resboxes[0] - assert len(metainterp.resboxes) == 2 - fs2 = [FakeFrame("code0", 0, b1t, c1, b2t), - FakeFrame("code1", 3, b3t, c2, b1t), - FakeFrame("code2", 10, c3, b2t, b4t)] - assert metainterp.framestack == fs2 - -def test_rebuild_from_resumedata_two_guards_w_shared_virtuals(): - py.test.skip("XXX rewrite") - b1, b2, b3, b4, b5, b6 = [InputArgRef(), InputArgRef(), BoxInt(), InputArgRef(), BoxInt(), BoxInt()] - c1, c2, c3, c4 = [ConstInt(1), ConstInt(2), ConstInt(3), - LLtypeMixin.nodebox.constbox()] - storage = Storage() - fs = [FakeFrame("code0", 0, c1, b2, b3)] - capture_resumedata(fs, None, [], storage) - - memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - values = {b2: virtual_value(b2, b5, c4)} - modifier = ResumeDataVirtualAdder(FakeOptimizer(values), storage, memo) - liveboxes = modifier.finish() - assert len(storage.rd_virtuals) == 1 - assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), - tag(0, TAGCONST)] - - storage2 = Storage() - fs = [FakeFrame("code0", 0, b1, b4, b2)] - capture_resumedata(fs, None, [], storage2) - values[b4] = virtual_value(b4, b6, c4) - modifier = ResumeDataVirtualAdder(FakeOptimizer(values), storage2, memo) - liveboxes = modifier.finish() - assert len(storage2.rd_virtuals) == 2 - assert storage2.rd_virtuals[1].fieldnums == storage.rd_virtuals[0].fieldnums - assert storage2.rd_virtuals[1] is storage.rd_virtuals[0] - - -def test_resumedata_top_recursive_virtuals(): - py.test.skip("XXX rewrite") - b1, b2, b3 = [InputArgRef(), InputArgRef(), BoxInt()] - storage = Storage() - fs = [FakeFrame("code0", 0, b1, b2)] - capture_resumedata(fs, None, [], storage) - - memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - v1 = virtual_value(b1, b3, None) - v2 = virtual_value(b2, b3, v1) - v1.setfield(LLtypeMixin.nextdescr, v2) - values = {b1: v1, b2: v2} - modifier = ResumeDataVirtualAdder(FakeOptimizer(values), storage, memo) - liveboxes = modifier.finish() - assert liveboxes == [b3] - assert len(storage.rd_virtuals) == 2 - assert storage.rd_virtuals[0].fieldnums == [tag(-1, TAGBOX), - tag(1, TAGVIRTUAL)] - assert storage.rd_virtuals[1].fieldnums == [tag(-1, TAGBOX), - tag(0, TAGVIRTUAL)] - - -# ____________________________________________________________ - def test_ResumeDataLoopMemo_ints(): memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) From pypy.commits at gmail.com Tue Jun 4 07:53:00 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Jun 2019 04:53:00 -0700 (PDT) Subject: [pypy-commit] pypy optimizeopt-cleanup: clean up dead code Message-ID: <5cf65b9c.1c69fb81.d5a91.f74e@mx.google.com> Author: Ronan Lamy Branch: optimizeopt-cleanup Changeset: r96735:de70eeda44e7 Date: 2019-05-28 18:42 +0100 http://bitbucket.org/pypy/pypy/changeset/de70eeda44e7/ Log: clean up dead code diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -534,7 +534,7 @@ def _force_elements(self, op, optforce, descr): # XXX descr = op.getdescr() - const = optforce.new_const_item(self.descr) + const = optforce.optimizer.new_const_item(self.descr) for i in range(self.length): item = self._items[i] if item is None: diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -166,18 +166,6 @@ def get_constant_box(self, box): return self.optimizer.get_constant_box(box) - def new_box(self, fieldofs): - return self.optimizer.new_box(fieldofs) - - def new_const(self, fieldofs): - return self.optimizer.new_const(fieldofs) - - def new_box_item(self, arraydescr): - return self.optimizer.new_box_item(arraydescr) - - def new_const_item(self, arraydescr): - return self.optimizer.new_const_item(arraydescr) - def pure(self, opnum, result): if self.optimizer.optpure: self.optimizer.optpure.pure(opnum, result) @@ -357,14 +345,6 @@ return None #self.ensure_imported(value) - def get_newoperations(self): - self.flush() - return self._newoperations - - def clear_newoperations(self): - self._newoperations = [] - self._emittedoperations = {} - def make_equal_to(self, op, newop): op = get_box_replacement(op) if op is newop: @@ -623,32 +603,6 @@ self._last_guard_op = None return op - def potentially_change_ovf_op_to_no_ovf(self, op): - # if last emitted operations was int_xxx_ovf and we are not emitting - # a guard_no_overflow change to int_add - if op.getopnum() != rop.GUARD_NO_OVERFLOW: - return - if not self._newoperations: - # got optimized otherwise - return - op = self._newoperations[-1] - if not op.is_ovf(): - return - newop = self.replace_op_with_no_ovf(op) - self._newoperations[-1] = newop - self._emittedoperations[newop] = None - - def replace_op_with_no_ovf(self, op): - if op.getopnum() == rop.INT_MUL_OVF: - return self.replace_op_with(op, rop.INT_MUL) - elif op.getopnum() == rop.INT_ADD_OVF: - return self.replace_op_with(op, rop.INT_ADD) - elif op.getopnum() == rop.INT_SUB_OVF: - return self.replace_op_with(op, rop.INT_SUB) - else: - assert False - - def _copy_resume_data_from(self, guard_op, last_guard_op): last_descr = last_guard_op.getdescr() descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, last_descr) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -28,7 +28,7 @@ assert clear opinfo = info.ArrayStructInfo(arraydescr, size, is_virtual=True) else: - const = self.new_const_item(arraydescr) + const = self.optimizer.new_const_item(arraydescr) opinfo = info.ArrayPtrInfo(arraydescr, const, size, clear, is_virtual=True) # Replace 'source_op' with a version in which the length is @@ -393,7 +393,7 @@ if fld is None: raise Exception("I think this is illegal") xxx - fieldvalue = self.new_const(descr) + fieldvalue = self.optimizer.new_const(descr) self.make_equal_to(op, fld) return self.make_nonnull(op.getarg(0)) From pypy.commits at gmail.com Tue Jun 4 07:53:06 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Jun 2019 04:53:06 -0700 (PDT) Subject: [pypy-commit] pypy optimizeopt-cleanup: hg merge default Message-ID: <5cf65ba2.1c69fb81.acb44.229d@mx.google.com> Author: Ronan Lamy Branch: optimizeopt-cleanup Changeset: r96736:7b5e54947ab1 Date: 2019-06-04 12:51 +0100 http://bitbucket.org/pypy/pypy/changeset/7b5e54947ab1/ Log: hg merge default diff too long, truncating to 2000 out of 13694 lines diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py --- a/extra_tests/cffi_tests/cffi0/test_parsing.py +++ b/extra_tests/cffi_tests/cffi0/test_parsing.py @@ -410,7 +410,17 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; + enum Enum { + POS = +1, + TWO = 2, + NIL = 0, + NEG = -1, + ADDSUB = (POS+TWO)-1, + DIVMULINT = (3 * 3) / 2, + SHIFT = (1 << 3) >> 1, + BINOPS = (0x7 & 0x1) | 0x8, + XOR = 0xf ^ 0xa + }; """) needs_dlopen_none() C = ffi.dlopen(None) @@ -418,7 +428,11 @@ assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 - assert C.OP == 2 + assert C.ADDSUB == 2 + assert C.DIVMULINT == 4 + assert C.SHIFT == 4 + assert C.BINOPS == 0b1001 + assert C.XOR == 0b0101 def test_stdcall(): ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py --- a/extra_tests/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2535,3 +2535,29 @@ x.p = p x.cyclic = x del p, x + +def test_arithmetic_in_cdef(): + for a in [0, 11, 15]: + ffi = FFI() + ffi.cdef(""" + enum FOO { + DIVNN = ((-?) / (-3)), + DIVNP = ((-?) / (+3)), + DIVPN = ((+?) / (-3)), + MODNN = ((-?) % (-3)), + MODNP = ((-?) % (+3)), + MODPN = ((+?) % (-3)), + }; + """.replace('?', str(a))) + lib = ffi.verify(""" + enum FOO { + DIVNN = ((-?) / (-3)), + DIVNP = ((-?) / (+3)), + DIVPN = ((+?) / (-3)), + MODNN = ((-?) % (-3)), + MODNP = ((-?) % (+3)), + MODPN = ((+?) % (-3)), + }; + """.replace('?', str(a))) + # the verify() crashes if the values in the enum are different from + # the values we computed ourselves from the cdef() diff --git a/extra_tests/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/extra_tests/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -2,11 +2,13 @@ import py, os, sys, shutil import subprocess from extra_tests.cffi_tests.udir import udir +import pytest if sys.platform == 'win32': - py.test.skip('snippets do not run on win32') + pytestmark = pytest.mark.skip('snippets do not run on win32') if sys.version_info < (2, 7): - py.test.skip('fails e.g. on a Debian/Ubuntu which patches virtualenv' + pytestmark = pytest.mark.skip( + 'fails e.g. on a Debian/Ubuntu which patches virtualenv' ' in a non-2.6-friendly way') def create_venv(name): diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -2339,3 +2339,24 @@ typedef int foo_t; struct foo_s { void (*x)(foo_t); }; """) py.test.raises(TypeError, ffi.new, "struct foo_s *") + +def test_from_buffer_struct(): + ffi = FFI() + ffi.cdef("""struct foo_s { int a, b; };""") + lib = verify(ffi, "test_from_buffer_struct_p", """ + struct foo_s { int a, b; }; + """) + p = ffi.new("struct foo_s *", [-219239, 58974983]) + q = ffi.from_buffer("struct foo_s[]", ffi.buffer(p)) + assert ffi.typeof(q) == ffi.typeof("struct foo_s[]") + assert len(q) == 1 + assert q[0].a == p.a + assert q[0].b == p.b + assert q == p + q = ffi.from_buffer("struct foo_s *", ffi.buffer(p)) + assert ffi.typeof(q) == ffi.typeof("struct foo_s *") + assert q.a == p.a + assert q.b == p.b + assert q[0].a == p.a + assert q[0].b == p.b + assert q == p diff --git a/extra_tests/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py --- a/extra_tests/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -64,8 +64,8 @@ output = popen.stdout.read() err = popen.wait() if err: - raise OSError("popen failed with exit code %r: %r" % ( - err, args)) + raise OSError(("popen failed with exit code %r: %r\n\n%s" % ( + err, args, output)).rstrip()) print(output.rstrip()) return output diff --git a/extra_tests/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py --- a/extra_tests/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -3,8 +3,8 @@ from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': - import py - py.test.skip("written with POSIX functions") + import pytest + pytestmark = pytest.mark.skip("written with POSIX functions") class TestPerformance(EmbeddingTests): diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py --- a/lib-python/2.7/ctypes/test/test_byteswap.py +++ b/lib-python/2.7/ctypes/test/test_byteswap.py @@ -2,7 +2,6 @@ from binascii import hexlify from ctypes import * -from ctypes.test import xfail def bin(s): return hexlify(memoryview(s)).upper() diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -71,7 +71,7 @@ g['AR'] = "ar" g['ARFLAGS'] = "rc" g['EXE'] = "" - g['LIBDIR'] = os.path.join(sys.prefix, 'lib') + g['LIBDIR'] = os.path.join(sys.prefix, 'bin') # where is the shared library g['VERSION'] = get_python_version() if sys.platform[:6] == "darwin": diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py --- a/lib-python/2.7/socket.py +++ b/lib-python/2.7/socket.py @@ -61,20 +61,22 @@ DeprecationWarning, stacklevel=2) return _realssl.sslwrap_simple(sock, keyfile, certfile) - # we need to import the same constants we used to... - from _ssl import SSLError as sslerror - from _ssl import \ - RAND_add, \ - RAND_status, \ - SSL_ERROR_ZERO_RETURN, \ - SSL_ERROR_WANT_READ, \ - SSL_ERROR_WANT_WRITE, \ - SSL_ERROR_WANT_X509_LOOKUP, \ - SSL_ERROR_SYSCALL, \ - SSL_ERROR_SSL, \ - SSL_ERROR_WANT_CONNECT, \ - SSL_ERROR_EOF, \ - SSL_ERROR_INVALID_ERROR_CODE + # we need to import the same constants we used to, + # see lib_pypy/_cffi_ssl/_stdssl/error.py and __init__.py to prevent + # circular import + # from _ssl import SSLError as sslerror + # from _ssl import \ + # RAND_add, \ + # RAND_status + # SSL_ERROR_ZERO_RETURN, \ + # SSL_ERROR_WANT_READ, \ + # SSL_ERROR_WANT_WRITE, \ + # SSL_ERROR_WANT_X509_LOOKUP, \ + # SSL_ERROR_SYSCALL, \ + # SSL_ERROR_SSL, \ + # SSL_ERROR_WANT_CONNECT, \ + # SSL_ERROR_EOF, \ + # SSL_ERROR_INVALID_ERROR_CODE try: from _ssl import RAND_egd except ImportError: diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -483,6 +483,7 @@ _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['implementation'] = _get_implementation() _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() + _CONFIG_VARS['LIBRARY'] = '' if os.name in ('nt', 'os2'): _init_non_posix(_CONFIG_VARS) diff --git a/lib-python/2.7/test/capath/efa5f9c3.0 b/lib-python/2.7/test/capath/efa5f9c3.0 new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/capath/efa5f9c3.0 @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL +BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG +A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw +HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx +FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu +aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD +DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2 +RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix +IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx +jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK +DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3 +TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI +aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU +OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH +OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg +7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ +8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB +AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW +gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP +ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN +9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9 +1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH +AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e +W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk +BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8 +XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku +B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL +Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV +J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg== +-----END CERTIFICATE----- diff --git a/lib-python/2.7/test/selfsigned_pythontestdotnet.pem b/lib-python/2.7/test/selfsigned_pythontestdotnet.pem --- a/lib-python/2.7/test/selfsigned_pythontestdotnet.pem +++ b/lib-python/2.7/test/selfsigned_pythontestdotnet.pem @@ -1,16 +1,34 @@ -----BEGIN CERTIFICATE----- -MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv -bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG -A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 -aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ -Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm -Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv -EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl -bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN -AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h -TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 -C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL +BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG +A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw +HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx +FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu +aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD +DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2 +RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix +IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx +jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK +DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3 +TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI +aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU +OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH +OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg +7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ +8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB +AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW +gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP +ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN +9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9 +1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH +AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e +W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk +BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8 +XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku +B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL +Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV +J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg== -----END CERTIFICATE----- diff --git a/lib-python/2.7/test/test_ftplib.py b/lib-python/2.7/test/test_ftplib.py --- a/lib-python/2.7/test/test_ftplib.py +++ b/lib-python/2.7/test/test_ftplib.py @@ -234,11 +234,17 @@ def run(self): self.active = True self.__flag.set() - while self.active and asyncore.socket_map: - self.active_lock.acquire() - asyncore.loop(timeout=0.1, count=1) - self.active_lock.release() - asyncore.close_all(ignore_all=True) + try: + while self.active and asyncore.socket_map: + self.active_lock.acquire() + try: + asyncore.loop(timeout=0.1, count=1) + except: + self.active_lock.release() + raise + self.active_lock.release() + finally: + asyncore.close_all(ignore_all=True) def stop(self): assert self.active diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py --- a/lib-python/2.7/test/test_ssl.py +++ b/lib-python/2.7/test/test_ssl.py @@ -764,12 +764,17 @@ ctx.set_ciphers("^$:,;?*'dorothyx") @skip_if_broken_ubuntu_ssl - def test_options(self): + def _test_options(self): + ''' + Disable this test, it is too flaky. Different platforms define + different defaults + ''' ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0): default |= ssl.OP_NO_COMPRESSION + default |= ssl.OP_ENABLE_MIDDLEBOX_COMPAT self.assertEqual(default, ctx.options) ctx.options |= ssl.OP_NO_TLSv1 self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) diff --git a/lib-python/2.7/test/test_sys.py b/lib-python/2.7/test/test_sys.py --- a/lib-python/2.7/test/test_sys.py +++ b/lib-python/2.7/test/test_sys.py @@ -216,6 +216,11 @@ raise ValueError() except ValueError, e: pass + except MemoryError: + # Documentation for setrecursionlimit says: "The highest possible + # limit is platform-dependent. ... a too-high limit can lead to a + # crash" so we allow MemoryError here + pass finally: sys.setrecursionlimit(oldlimit) diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py --- a/lib-python/2.7/test/test_unicode.py +++ b/lib-python/2.7/test/test_unicode.py @@ -1652,10 +1652,10 @@ # when a string allocation fails with a MemoryError. # This used to crash the interpreter, # or leak references when the number was smaller. - charwidth = 4 if sys.maxunicode >= 0x10000 else 2 + charwidth = 2 # pypy: the char \u0123 is stored in two utf-8 bytes # Note: sys.maxsize is half of the actual max allocation because of # the signedness of Py_ssize_t. - alloc = lambda: u"a" * (sys.maxsize // charwidth * 2) + alloc = lambda: u"\u0123" * (sys.maxsize // charwidth * 2) self.assertRaises(MemoryError, alloc) self.assertRaises(MemoryError, alloc) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -176,7 +176,7 @@ RegrTest('test_copy_reg.py', core=True), RegrTest('test_cpickle.py', core=True), RegrTest('test_cprofile.py'), - RegrTest('test_crypt.py', usemodules='crypt'), + RegrTest('test_crypt.py'), RegrTest('test_csv.py', usemodules='_csv'), RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"), RegrTest('test_curses.py'), @@ -401,7 +401,7 @@ RegrTest('test_source_encoding.py'), RegrTest('test_spwd.py'), RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), - RegrTest('test_ssl.py', usemodules='_ssl _socket select'), + RegrTest('test_ssl.py', usemodules='_socket select'), RegrTest('test_startfile.py'), RegrTest('test_stat.py'), RegrTest('test_str.py', core=True), diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/LICENSE @@ -0,0 +1,26 @@ + +Except when otherwise stated (look for LICENSE files in directories or +information at the beginning of each file) all software and +documentation is licensed as follows: + + The MIT License + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/README.md @@ -0,0 +1,35 @@ +# PyPy's SSL module + +All of the CFFI code is copied from cryptography, wich patches contributed +back to cryptography. PyPy vendors it's own copy of the cffi backend thus +it renames the compiled shared object to _pypy_openssl.so (which means +that cryptography can ship their own cffi backend) + +NOTE: currently, we have the following changes: + +* ``_cffi_src/openssl/callbacks.py`` to not rely on the CPython C API + (this change is now backported) + +* ``_cffi_src/utils.py`` for issue #2575 (29c9a89359e4) + +* ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1) + +* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates() + + +# Tests? + +Currently this module is tested using CPython's standard library test suite. + +# Install it into PyPy's source tree + +Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command:: + + $ cp -r /src/_cffi_src/* . + +NOTE: you need to keep our version of ``_cffi_src/openssl/callbacks.py`` +for now! + +# Crpytography version + +Copied over release version `1.7.2` diff --git a/lib_pypy/_cffi_ssl/__init__.py b/lib_pypy/_cffi_ssl/__init__.py new file mode 100644 diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py new file mode 100644 diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py @@ -0,0 +1,33 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from _cffi_src.utils import build_ffi_for_binding + + +ffi = build_ffi_for_binding( + module_name="_commoncrypto", + module_prefix="_cffi_src.commoncrypto.", + modules=[ + "cf", + "common_digest", + "common_hmac", + "common_key_derivation", + "common_cryptor", + "common_symmetric_key_wrap", + "seccertificate", + "secimport", + "secitem", + "seckey", + "seckeychain", + "secpolicy", + "sectransform", + "sectrust", + "secure_transport", + ], + extra_link_args=[ + "-framework", "Security", "-framework", "CoreFoundation" + ], +) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py @@ -0,0 +1,27 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import os + +from _cffi_src.utils import build_ffi, compiler_type, extra_link_args + + +with open(os.path.join( + os.path.dirname(__file__), "hazmat_src/constant_time.h" +)) as f: + types = f.read() + +with open(os.path.join( + os.path.dirname(__file__), "hazmat_src/constant_time.c" +)) as f: + functions = f.read() + +ffi = build_ffi( + module_name="_constant_time", + cdef_source=types, + verify_source=functions, + extra_link_args=extra_link_args(compiler_type()), +) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py @@ -0,0 +1,86 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import os +import sys + +from _cffi_src.utils import ( + build_ffi_for_binding, compiler_type, extra_link_args +) + + +def _get_openssl_libraries(platform): + # OpenSSL goes by a different library name on different operating systems. + if platform == "darwin": + return _osx_libraries( + os.environ.get("CRYPTOGRAPHY_OSX_NO_LINK_FLAGS") + ) + elif platform == "win32": + if compiler_type() == "msvc": + libs = ["libeay32", "ssleay32"] + else: + libs = ["ssl", "crypto"] + return libs + ["advapi32", "crypt32", "gdi32", "user32", "ws2_32"] + else: + # In some circumstances, the order in which these libs are + # specified on the linker command-line is significant; + # libssl must come before libcrypto + # (http://marc.info/?l=openssl-users&m=135361825921871) + return ["ssl", "crypto"] + + +def _osx_libraries(build_static): + # For building statically we don't want to pass the -lssl or -lcrypto flags + if build_static == "1": + return [] + else: + return ["ssl", "crypto"] + + +ffi = build_ffi_for_binding( + module_name="_openssl", + module_prefix="_cffi_src.openssl.", + modules=[ + # This goes first so we can define some cryptography-wide symbols. + "cryptography", + + "aes", + "asn1", + "bignum", + "bio", + "cmac", + "cms", + "conf", + "crypto", + "dh", + "dsa", + "ec", + "ecdh", + "ecdsa", + "engine", + "err", + "evp", + "hmac", + "nid", + "objects", + "ocsp", + "opensslv", + "osrandom_engine", + "pem", + "pkcs12", + "rand", + "rsa", + "ssl", + "x509", + "x509name", + "x509v3", + "x509_vfy", + "pkcs7", + "callbacks", + ], + libraries=_get_openssl_libraries(sys.platform), + extra_link_args=extra_link_args(compiler_type()), +) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/build_padding.py @@ -0,0 +1,27 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import os + +from _cffi_src.utils import build_ffi, compiler_type, extra_link_args + + +with open(os.path.join( + os.path.dirname(__file__), "hazmat_src/padding.h" +)) as f: + types = f.read() + +with open(os.path.join( + os.path.dirname(__file__), "hazmat_src/padding.c" +)) as f: + functions = f.read() + +ffi = build_ffi( + module_name="_padding", + cdef_source=types, + verify_source=functions, + extra_link_args=extra_link_args(compiler_type()), +) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/cf.py @@ -0,0 +1,113 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef bool Boolean; +typedef signed long OSStatus; +typedef unsigned char UInt8; +typedef uint32_t UInt32; + +typedef const void * CFAllocatorRef; +const CFAllocatorRef kCFAllocatorDefault; +typedef ... *CFDataRef; +typedef signed long long CFIndex; +typedef ... *CFStringRef; +typedef ... *CFArrayRef; +typedef ... *CFMutableArrayRef; +typedef ... *CFBooleanRef; +typedef ... *CFErrorRef; +typedef ... *CFNumberRef; +typedef ... *CFTypeRef; +typedef ... *CFDictionaryRef; +typedef ... *CFMutableDictionaryRef; +typedef struct { + ...; +} CFDictionaryKeyCallBacks; +typedef struct { + ...; +} CFDictionaryValueCallBacks; +typedef struct { + ...; +} CFRange; +typedef struct { + ...; +} CFArrayCallBacks; + +typedef UInt32 CFStringEncoding; +enum { + kCFStringEncodingASCII = 0x0600 +}; + +enum { + kCFNumberSInt8Type = 1, + kCFNumberSInt16Type = 2, + kCFNumberSInt32Type = 3, + kCFNumberSInt64Type = 4, + kCFNumberFloat32Type = 5, + kCFNumberFloat64Type = 6, + kCFNumberCharType = 7, + kCFNumberShortType = 8, + kCFNumberIntType = 9, + kCFNumberLongType = 10, + kCFNumberLongLongType = 11, + kCFNumberFloatType = 12, + kCFNumberDoubleType = 13, + kCFNumberCFIndexType = 14, + kCFNumberNSIntegerType = 15, + kCFNumberCGFloatType = 16, + kCFNumberMaxType = 16 +}; +typedef int CFNumberType; + +const CFDictionaryKeyCallBacks kCFTypeDictionaryKeyCallBacks; +const CFDictionaryValueCallBacks kCFTypeDictionaryValueCallBacks; + +const CFArrayCallBacks kCFTypeArrayCallBacks; + +const CFBooleanRef kCFBooleanTrue; +const CFBooleanRef kCFBooleanFalse; +""" + +FUNCTIONS = """ +CFDataRef CFDataCreate(CFAllocatorRef, const UInt8 *, CFIndex); +CFStringRef CFStringCreateWithCString(CFAllocatorRef, const char *, + CFStringEncoding); +CFDictionaryRef CFDictionaryCreate(CFAllocatorRef, const void **, + const void **, CFIndex, + const CFDictionaryKeyCallBacks *, + const CFDictionaryValueCallBacks *); +CFMutableDictionaryRef CFDictionaryCreateMutable( + CFAllocatorRef, + CFIndex, + const CFDictionaryKeyCallBacks *, + const CFDictionaryValueCallBacks * +); +void CFDictionarySetValue(CFMutableDictionaryRef, const void *, const void *); +CFIndex CFArrayGetCount(CFArrayRef); +const void *CFArrayGetValueAtIndex(CFArrayRef, CFIndex); +CFIndex CFDataGetLength(CFDataRef); +void CFDataGetBytes(CFDataRef, CFRange, UInt8 *); +CFRange CFRangeMake(CFIndex, CFIndex); +void CFShow(CFTypeRef); +Boolean CFBooleanGetValue(CFBooleanRef); +CFNumberRef CFNumberCreate(CFAllocatorRef, CFNumberType, const void *); +void CFRelease(CFTypeRef); +CFTypeRef CFRetain(CFTypeRef); + +CFMutableArrayRef CFArrayCreateMutable(CFAllocatorRef, CFIndex, + const CFArrayCallBacks *); +void CFArrayAppendValue(CFMutableArrayRef, const void *); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_cryptor.py @@ -0,0 +1,99 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +enum { + kCCAlgorithmAES128 = 0, + kCCAlgorithmDES, + kCCAlgorithm3DES, + kCCAlgorithmCAST, + kCCAlgorithmRC4, + kCCAlgorithmRC2, + kCCAlgorithmBlowfish +}; +typedef uint32_t CCAlgorithm; +enum { + kCCSuccess = 0, + kCCParamError = -4300, + kCCBufferTooSmall = -4301, + kCCMemoryFailure = -4302, + kCCAlignmentError = -4303, + kCCDecodeError = -4304, + kCCUnimplemented = -4305 +}; +typedef int32_t CCCryptorStatus; +typedef uint32_t CCOptions; +enum { + kCCEncrypt = 0, + kCCDecrypt, +}; +typedef uint32_t CCOperation; +typedef ... *CCCryptorRef; + +enum { + kCCModeOptionCTR_LE = 0x0001, + kCCModeOptionCTR_BE = 0x0002 +}; + +typedef uint32_t CCModeOptions; + +enum { + kCCModeECB = 1, + kCCModeCBC = 2, + kCCModeCFB = 3, + kCCModeCTR = 4, + kCCModeF8 = 5, + kCCModeLRW = 6, + kCCModeOFB = 7, + kCCModeXTS = 8, + kCCModeRC4 = 9, + kCCModeCFB8 = 10, + kCCModeGCM = 11 +}; +typedef uint32_t CCMode; +enum { + ccNoPadding = 0, + ccPKCS7Padding = 1, +}; +typedef uint32_t CCPadding; +""" + +FUNCTIONS = """ +CCCryptorStatus CCCryptorCreateWithMode(CCOperation, CCMode, CCAlgorithm, + CCPadding, const void *, const void *, + size_t, const void *, size_t, int, + CCModeOptions, CCCryptorRef *); +CCCryptorStatus CCCryptorCreate(CCOperation, CCAlgorithm, CCOptions, + const void *, size_t, const void *, + CCCryptorRef *); +CCCryptorStatus CCCryptorUpdate(CCCryptorRef, const void *, size_t, void *, + size_t, size_t *); +CCCryptorStatus CCCryptorFinal(CCCryptorRef, void *, size_t, size_t *); +CCCryptorStatus CCCryptorRelease(CCCryptorRef); + +CCCryptorStatus CCCryptorGCMAddIV(CCCryptorRef, const void *, size_t); +CCCryptorStatus CCCryptorGCMAddAAD(CCCryptorRef, const void *, size_t); +CCCryptorStatus CCCryptorGCMEncrypt(CCCryptorRef, const void *, size_t, + void *); +CCCryptorStatus CCCryptorGCMDecrypt(CCCryptorRef, const void *, size_t, + void *); +CCCryptorStatus CCCryptorGCMFinal(CCCryptorRef, const void *, size_t *); +CCCryptorStatus CCCryptorGCMReset(CCCryptorRef); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +/* Not defined in the public header */ +enum { + kCCModeGCM = 11 +}; +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_digest.py @@ -0,0 +1,58 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef uint32_t CC_LONG; +typedef uint64_t CC_LONG64; +typedef struct CC_MD5state_st { + ...; +} CC_MD5_CTX; +typedef struct CC_SHA1state_st { + ...; +} CC_SHA1_CTX; +typedef struct CC_SHA256state_st { + ...; +} CC_SHA256_CTX; +typedef struct CC_SHA512state_st { + ...; +} CC_SHA512_CTX; +""" + +FUNCTIONS = """ +int CC_MD5_Init(CC_MD5_CTX *); +int CC_MD5_Update(CC_MD5_CTX *, const void *, CC_LONG); +int CC_MD5_Final(unsigned char *, CC_MD5_CTX *); + +int CC_SHA1_Init(CC_SHA1_CTX *); +int CC_SHA1_Update(CC_SHA1_CTX *, const void *, CC_LONG); +int CC_SHA1_Final(unsigned char *, CC_SHA1_CTX *); + +int CC_SHA224_Init(CC_SHA256_CTX *); +int CC_SHA224_Update(CC_SHA256_CTX *, const void *, CC_LONG); +int CC_SHA224_Final(unsigned char *, CC_SHA256_CTX *); + +int CC_SHA256_Init(CC_SHA256_CTX *); +int CC_SHA256_Update(CC_SHA256_CTX *, const void *, CC_LONG); +int CC_SHA256_Final(unsigned char *, CC_SHA256_CTX *); + +int CC_SHA384_Init(CC_SHA512_CTX *); +int CC_SHA384_Update(CC_SHA512_CTX *, const void *, CC_LONG); +int CC_SHA384_Final(unsigned char *, CC_SHA512_CTX *); + +int CC_SHA512_Init(CC_SHA512_CTX *); +int CC_SHA512_Update(CC_SHA512_CTX *, const void *, CC_LONG); +int CC_SHA512_Final(unsigned char *, CC_SHA512_CTX *); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_hmac.py @@ -0,0 +1,37 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef struct { + ...; +} CCHmacContext; +enum { + kCCHmacAlgSHA1, + kCCHmacAlgMD5, + kCCHmacAlgSHA256, + kCCHmacAlgSHA384, + kCCHmacAlgSHA512, + kCCHmacAlgSHA224 +}; +typedef uint32_t CCHmacAlgorithm; +""" + +FUNCTIONS = """ +void CCHmacInit(CCHmacContext *, CCHmacAlgorithm, const void *, size_t); +void CCHmacUpdate(CCHmacContext *, const void *, size_t); +void CCHmacFinal(CCHmacContext *, void *); + +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_key_derivation.py @@ -0,0 +1,39 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +enum { + kCCPBKDF2 = 2, +}; +typedef uint32_t CCPBKDFAlgorithm; +enum { + kCCPRFHmacAlgSHA1 = 1, + kCCPRFHmacAlgSHA224 = 2, + kCCPRFHmacAlgSHA256 = 3, + kCCPRFHmacAlgSHA384 = 4, + kCCPRFHmacAlgSHA512 = 5, +}; +typedef uint32_t CCPseudoRandomAlgorithm; +typedef unsigned int uint; +""" + +FUNCTIONS = """ +int CCKeyDerivationPBKDF(CCPBKDFAlgorithm, const char *, size_t, + const uint8_t *, size_t, CCPseudoRandomAlgorithm, + uint, uint8_t *, size_t); +uint CCCalibratePBKDF(CCPBKDFAlgorithm, size_t, size_t, + CCPseudoRandomAlgorithm, size_t, uint32_t); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/common_symmetric_key_wrap.py @@ -0,0 +1,35 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +enum { + kCCWRAPAES = 1, +}; + +typedef uint32_t CCWrappingAlgorithm; +""" + +FUNCTIONS = """ +int CCSymmetricKeyWrap(CCWrappingAlgorithm, const uint8_t *, const size_t, + const uint8_t *, size_t, const uint8_t *, size_t, + uint8_t *, size_t *); +int CCSymmetricKeyUnwrap(CCWrappingAlgorithm algorithm, const uint8_t *, + const size_t, const uint8_t *, size_t, + const uint8_t *, size_t, uint8_t *, size_t *); +size_t CCSymmetricWrappedSize(CCWrappingAlgorithm, size_t); +size_t CCSymmetricUnwrappedSize(CCWrappingAlgorithm, size_t); + +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seccertificate.py @@ -0,0 +1,23 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *SecCertificateRef; +""" + +FUNCTIONS = """ +SecCertificateRef SecCertificateCreateWithData(CFAllocatorRef, CFDataRef); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secimport.py @@ -0,0 +1,86 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *SecAccessRef; + +CFStringRef kSecImportExportPassphrase; +CFStringRef kSecImportExportKeychain; +CFStringRef kSecImportExportAccess; + +typedef uint32_t SecExternalItemType; +enum { + kSecItemTypeUnknown, + kSecItemTypePrivateKey, + kSecItemTypePublicKey, + kSecItemTypeSessionKey, + kSecItemTypeCertificate, + kSecItemTypeAggregate +}; + + +typedef uint32_t SecExternalFormat; +enum { + kSecFormatUnknown = 0, + kSecFormatOpenSSL, + kSecFormatSSH, + kSecFormatBSAFE, + kSecFormatRawKey, + kSecFormatWrappedPKCS8, + kSecFormatWrappedOpenSSL, + kSecFormatWrappedSSH, + kSecFormatWrappedLSH, + kSecFormatX509Cert, + kSecFormatPEMSequence, + kSecFormatPKCS7, + kSecFormatPKCS12, + kSecFormatNetscapeCertSequence, + kSecFormatSSHv2 +}; + +typedef uint32_t SecItemImportExportFlags; +enum { + kSecKeyImportOnlyOne = 0x00000001, + kSecKeySecurePassphrase = 0x00000002, + kSecKeyNoAccessControl = 0x00000004 +}; +typedef uint32_t SecKeyImportExportFlags; + +typedef struct { + /* for import and export */ + uint32_t version; + SecKeyImportExportFlags flags; + CFTypeRef passphrase; + CFStringRef alertTitle; + CFStringRef alertPrompt; + + /* for import only */ + SecAccessRef accessRef; + CFArrayRef keyUsage; + + CFArrayRef keyAttributes; +} SecItemImportExportKeyParameters; +""" + +FUNCTIONS = """ +OSStatus SecItemImport(CFDataRef, CFStringRef, SecExternalFormat *, + SecExternalItemType *, SecItemImportExportFlags, + const SecItemImportExportKeyParameters *, + SecKeychainRef, CFArrayRef *); +OSStatus SecPKCS12Import(CFDataRef, CFDictionaryRef, CFArrayRef *); +OSStatus SecItemExport(CFTypeRef, SecExternalFormat, SecItemImportExportFlags, + const SecItemImportExportKeyParameters *, CFDataRef *); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secitem.py @@ -0,0 +1,27 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +const CFTypeRef kSecAttrKeyType; +const CFTypeRef kSecAttrKeySizeInBits; +const CFTypeRef kSecAttrIsPermanent; +const CFTypeRef kSecAttrKeyTypeRSA; +const CFTypeRef kSecAttrKeyTypeDSA; +const CFTypeRef kSecUseKeychain; +""" + +FUNCTIONS = """ +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckey.py @@ -0,0 +1,24 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *SecKeyRef; +""" + +FUNCTIONS = """ +OSStatus SecKeyGeneratePair(CFDictionaryRef, SecKeyRef *, SecKeyRef *); +size_t SecKeyGetBlockSize(SecKeyRef); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/seckeychain.py @@ -0,0 +1,25 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *SecKeychainRef; +""" + +FUNCTIONS = """ +OSStatus SecKeychainCreate(const char *, UInt32, const void *, Boolean, + SecAccessRef, SecKeychainRef *); +OSStatus SecKeychainDelete(SecKeychainRef); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secpolicy.py @@ -0,0 +1,23 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *SecPolicyRef; +""" + +FUNCTIONS = """ +SecPolicyRef SecPolicyCreateSSL(Boolean, CFStringRef); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectransform.py @@ -0,0 +1,68 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +#include +#include +""" + +TYPES = """ +typedef ... *SecTransformRef; + +CFStringRef kSecImportExportPassphrase; +CFStringRef kSecImportExportKeychain; +CFStringRef kSecImportExportAccess; + +CFStringRef kSecEncryptionMode; +CFStringRef kSecEncryptKey; +CFStringRef kSecIVKey; +CFStringRef kSecModeCBCKey; +CFStringRef kSecModeCFBKey; +CFStringRef kSecModeECBKey; +CFStringRef kSecModeNoneKey; +CFStringRef kSecModeOFBKey; +CFStringRef kSecOAEPEncodingParametersAttributeName; +CFStringRef kSecPaddingKey; +CFStringRef kSecPaddingNoneKey; +CFStringRef kSecPaddingOAEPKey; +CFStringRef kSecPaddingPKCS1Key; +CFStringRef kSecPaddingPKCS5Key; +CFStringRef kSecPaddingPKCS7Key; + +const CFStringRef kSecTransformInputAttributeName; +const CFStringRef kSecTransformOutputAttributeName; +const CFStringRef kSecTransformDebugAttributeName; +const CFStringRef kSecTransformTransformName; +const CFStringRef kSecTransformAbortAttributeName; + +CFStringRef kSecInputIsAttributeName; +CFStringRef kSecInputIsPlainText; +CFStringRef kSecInputIsDigest; +CFStringRef kSecInputIsRaw; + +const CFStringRef kSecDigestTypeAttribute; +const CFStringRef kSecDigestLengthAttribute; +const CFStringRef kSecDigestMD5; +const CFStringRef kSecDigestSHA1; +const CFStringRef kSecDigestSHA2; +""" + +FUNCTIONS = """ +Boolean SecTransformSetAttribute(SecTransformRef, CFStringRef, CFTypeRef, + CFErrorRef *); +SecTransformRef SecDecryptTransformCreate(SecKeyRef, CFErrorRef *); +SecTransformRef SecEncryptTransformCreate(SecKeyRef, CFErrorRef *); +SecTransformRef SecVerifyTransformCreate(SecKeyRef, CFDataRef, CFErrorRef *); +SecTransformRef SecSignTransformCreate(SecKeyRef, CFErrorRef *) ; +CFTypeRef SecTransformExecute(SecTransformRef, CFErrorRef *); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/sectrust.py @@ -0,0 +1,39 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *SecTrustRef; +typedef uint32_t SecTrustResultType; + +enum { + kSecTrustResultInvalid, + kSecTrustResultProceed, + kSecTrustResultDeny, + kSecTrustResultUnspecified, + kSecTrustResultRecoverableTrustFailure, + kSecTrustResultFatalTrustFailure, + kSecTrustResultOtherError +}; +""" + +FUNCTIONS = """ +OSStatus SecTrustEvaluate(SecTrustRef, SecTrustResultType *); +OSStatus SecTrustCopyAnchorCertificates(CFArrayRef *); +""" + +MACROS = """ +/* The first argument changed from CFArrayRef to CFTypeRef in 10.8, so this + * has to go here for compatibility. + */ +OSStatus SecTrustCreateWithCertificates(CFTypeRef, CFTypeRef, SecTrustRef *); +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/commoncrypto/secure_transport.py @@ -0,0 +1,308 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef ... *SSLContextRef; +typedef const void *SSLConnectionRef; + +typedef enum { + kSSLSessionOptionBreakOnServerAuth, + kSSLSessionOptionBreakOnCertRequested, +} SSLSessionOption; + +typedef enum { + kNeverAuthenticate, + kAlwaysAuthenticate, + kTryAuthenticate +} SSLAuthenticate; + +typedef enum { + kSSLIdle, + kSSLHandshake, + kSSLConnected, + kSSLClosed, + kSSLAborted +} SSLSessionState; + +typedef enum { + kSSLProtocolUnknown = 0, + kSSLProtocol3 = 2, + kTLSProtocol1 = 4, + /* DEPRECATED on iOS */ + kSSLProtocol2 = 1, + kSSLProtocol3Only = 3, + kTLSProtocol1Only = 5, + kSSLProtocolAll = 6, +} SSLProtocol; + +typedef UInt32 SSLCipherSuite; +enum { + SSL_NULL_WITH_NULL_NULL = 0x0000, + SSL_RSA_WITH_NULL_MD5 = 0x0001, + SSL_RSA_WITH_NULL_SHA = 0x0002, + SSL_RSA_EXPORT_WITH_RC4_40_MD5 = 0x0003, + SSL_RSA_WITH_RC4_128_MD5 = 0x0004, + SSL_RSA_WITH_RC4_128_SHA = 0x0005, + SSL_RSA_EXPORT_WITH_RC2_CBC_40_MD5 = 0x0006, + SSL_RSA_WITH_IDEA_CBC_SHA = 0x0007, + SSL_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0008, + SSL_RSA_WITH_DES_CBC_SHA = 0x0009, + SSL_RSA_WITH_3DES_EDE_CBC_SHA = 0x000A, + SSL_DH_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x000B, + SSL_DH_DSS_WITH_DES_CBC_SHA = 0x000C, + SSL_DH_DSS_WITH_3DES_EDE_CBC_SHA = 0x000D, + SSL_DH_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x000E, + SSL_DH_RSA_WITH_DES_CBC_SHA = 0x000F, + SSL_DH_RSA_WITH_3DES_EDE_CBC_SHA = 0x0010, + SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011, + SSL_DHE_DSS_WITH_DES_CBC_SHA = 0x0012, + SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA = 0x0013, + SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014, + SSL_DHE_RSA_WITH_DES_CBC_SHA = 0x0015, + SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA = 0x0016, + SSL_DH_anon_EXPORT_WITH_RC4_40_MD5 = 0x0017, + SSL_DH_anon_WITH_RC4_128_MD5 = 0x0018, + SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019, + SSL_DH_anon_WITH_DES_CBC_SHA = 0x001A, + SSL_DH_anon_WITH_3DES_EDE_CBC_SHA = 0x001B, + SSL_FORTEZZA_DMS_WITH_NULL_SHA = 0x001C, + SSL_FORTEZZA_DMS_WITH_FORTEZZA_CBC_SHA =0x001D, + + /* TLS addenda using AES, per RFC 3268 */ + TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F, + TLS_DH_DSS_WITH_AES_128_CBC_SHA = 0x0030, + TLS_DH_RSA_WITH_AES_128_CBC_SHA = 0x0031, + TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032, + TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033, + TLS_DH_anon_WITH_AES_128_CBC_SHA = 0x0034, + TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035, + TLS_DH_DSS_WITH_AES_256_CBC_SHA = 0x0036, + TLS_DH_RSA_WITH_AES_256_CBC_SHA = 0x0037, + TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038, + TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039, + TLS_DH_anon_WITH_AES_256_CBC_SHA = 0x003A, + + /* ECDSA addenda, RFC 4492 */ + TLS_ECDH_ECDSA_WITH_NULL_SHA = 0xC001, + TLS_ECDH_ECDSA_WITH_RC4_128_SHA = 0xC002, + TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC003, + TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA = 0xC004, + TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA = 0xC005, + TLS_ECDHE_ECDSA_WITH_NULL_SHA = 0xC006, + TLS_ECDHE_ECDSA_WITH_RC4_128_SHA = 0xC007, + TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC008, + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009, + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A, + TLS_ECDH_RSA_WITH_NULL_SHA = 0xC00B, + TLS_ECDH_RSA_WITH_RC4_128_SHA = 0xC00C, + TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA = 0xC00D, + TLS_ECDH_RSA_WITH_AES_128_CBC_SHA = 0xC00E, + TLS_ECDH_RSA_WITH_AES_256_CBC_SHA = 0xC00F, + TLS_ECDHE_RSA_WITH_NULL_SHA = 0xC010, + TLS_ECDHE_RSA_WITH_RC4_128_SHA = 0xC011, + TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA = 0xC012, + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013, + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014, + TLS_ECDH_anon_WITH_NULL_SHA = 0xC015, + TLS_ECDH_anon_WITH_RC4_128_SHA = 0xC016, + TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA = 0xC017, + TLS_ECDH_anon_WITH_AES_128_CBC_SHA = 0xC018, + TLS_ECDH_anon_WITH_AES_256_CBC_SHA = 0xC019, + + /* TLS 1.2 addenda, RFC 5246 */ + /* Initial state. */ + TLS_NULL_WITH_NULL_NULL = 0x0000, + + /* Server provided RSA certificate for key exchange. */ + TLS_RSA_WITH_NULL_MD5 = 0x0001, + TLS_RSA_WITH_NULL_SHA = 0x0002, + TLS_RSA_WITH_RC4_128_MD5 = 0x0004, + TLS_RSA_WITH_RC4_128_SHA = 0x0005, + TLS_RSA_WITH_3DES_EDE_CBC_SHA = 0x000A, + TLS_RSA_WITH_NULL_SHA256 = 0x003B, + TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C, + TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D, + + /* Server-authenticated (and optionally client-authenticated) + Diffie-Hellman. */ + TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA = 0x000D, + TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA = 0x0010, + TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA = 0x0013, + TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA = 0x0016, + TLS_DH_DSS_WITH_AES_128_CBC_SHA256 = 0x003E, + TLS_DH_RSA_WITH_AES_128_CBC_SHA256 = 0x003F, + TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040, + TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067, + TLS_DH_DSS_WITH_AES_256_CBC_SHA256 = 0x0068, + TLS_DH_RSA_WITH_AES_256_CBC_SHA256 = 0x0069, + TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A, + TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B, + + /* Completely anonymous Diffie-Hellman */ + TLS_DH_anon_WITH_RC4_128_MD5 = 0x0018, + TLS_DH_anon_WITH_3DES_EDE_CBC_SHA = 0x001B, + TLS_DH_anon_WITH_AES_128_CBC_SHA256 = 0x006C, + TLS_DH_anon_WITH_AES_256_CBC_SHA256 = 0x006D, + + /* Addenda from rfc 5288 AES Galois Counter Mode (GCM) Cipher Suites + for TLS. */ + TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C, + TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D, + TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E, + TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F, + TLS_DH_RSA_WITH_AES_128_GCM_SHA256 = 0x00A0, + TLS_DH_RSA_WITH_AES_256_GCM_SHA384 = 0x00A1, + TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2, + TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3, + TLS_DH_DSS_WITH_AES_128_GCM_SHA256 = 0x00A4, + TLS_DH_DSS_WITH_AES_256_GCM_SHA384 = 0x00A5, + TLS_DH_anon_WITH_AES_128_GCM_SHA256 = 0x00A6, + TLS_DH_anon_WITH_AES_256_GCM_SHA384 = 0x00A7, + + /* Addenda from rfc 5289 Elliptic Curve Cipher Suites with + HMAC SHA-256/384. */ + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023, + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024, + TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC025, + TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC026, + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027, + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028, + TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 = 0xC029, + TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 = 0xC02A, + + /* Addenda from rfc 5289 Elliptic Curve Cipher Suites with + SHA-256/384 and AES Galois Counter Mode (GCM) */ + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B, + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C, + TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02D, + TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02E, + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F, + TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030, + TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 = 0xC031, + TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 = 0xC032, + + /* RFC 5746 - Secure Renegotiation */ + TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FF, + + /* + * Tags for SSL 2 cipher kinds that are not specified + * for SSL 3. + */ + SSL_RSA_WITH_RC2_CBC_MD5 = 0xFF80, + SSL_RSA_WITH_IDEA_CBC_MD5 = 0xFF81, + SSL_RSA_WITH_DES_CBC_MD5 = 0xFF82, + SSL_RSA_WITH_3DES_EDE_CBC_MD5 = 0xFF83, + SSL_NO_SUCH_CIPHERSUITE = 0xFFFF +}; + +typedef enum { + kSSLClientCertNone, + kSSLClientCertRequested, + kSSLClientCertSent, + kSSLClientCertRejected +} SSLClientCertificateState; + +enum { + errSSLProtocol = -9800, + errSSLNegotiation = -9801, + errSSLFatalAlert = -9802, + errSSLWouldBlock = -9803, + errSSLSessionNotFound = -9804, + errSSLClosedGraceful = -9805, + errSSLClosedAbort = -9806, + errSSLXCertChainInvalid = -9807, + errSSLBadCert = -9808, + errSSLCrypto = -9809, + errSSLInternal = -9810, + errSSLModuleAttach = -9811, + errSSLUnknownRootCert = -9812, + errSSLNoRootCert = -9813, + errSSLCertExpired = -9814, + errSSLCertNotYetValid = -9815, + errSSLClosedNoNotify = -9816, + errSSLBufferOverflow = -9817, + errSSLBadCipherSuite = -9818, + errSSLPeerUnexpectedMsg = -9819, + errSSLPeerBadRecordMac = -9820, + errSSLPeerDecryptionFail = -9821, + errSSLPeerRecordOverflow = -9822, + errSSLPeerDecompressFail = -9823, + errSSLPeerHandshakeFail = -9824, + errSSLPeerBadCert = -9825, + errSSLPeerUnsupportedCert = -9826, + errSSLPeerCertRevoked = -9827, + errSSLPeerCertExpired = -9828, + errSSLPeerCertUnknown = -9829, + errSSLIllegalParam = -9830, + errSSLPeerUnknownCA = -9831, + errSSLPeerAccessDenied = -9832, + errSSLPeerDecodeError = -9833, + errSSLPeerDecryptError = -9834, + errSSLPeerExportRestriction = -9835, + errSSLPeerProtocolVersion = -9836, + errSSLPeerInsufficientSecurity = -9837, + errSSLPeerInternalError = -9838, + errSSLPeerUserCancelled = -9839, + errSSLPeerNoRenegotiation = -9840, + errSSLServerAuthCompleted = -9841, + errSSLClientCertRequested = -9842, + errSSLHostNameMismatch = -9843, + errSSLConnectionRefused = -9844, + errSSLDecryptionFail = -9845, + errSSLBadRecordMac = -9846, + errSSLRecordOverflow = -9847, + errSSLBadConfiguration = -9848, + errSSLLast = -9849 /* end of range, to be deleted */ +}; +""" + +FUNCTIONS = """ +OSStatus SSLSetConnection(SSLContextRef, SSLConnectionRef); +OSStatus SSLGetConnection(SSLContextRef, SSLConnectionRef *); +OSStatus SSLSetSessionOption(SSLContextRef, SSLSessionOption, Boolean); +OSStatus SSLSetClientSideAuthenticate(SSLContextRef, SSLAuthenticate); + +OSStatus SSLHandshake(SSLContextRef); +OSStatus SSLGetSessionState(SSLContextRef, SSLSessionState *); +OSStatus SSLGetNegotiatedProtocolVersion(SSLContextRef, SSLProtocol *); +OSStatus SSLSetPeerID(SSLContextRef, const void *, size_t); +OSStatus SSLGetPeerID(SSLContextRef, const void **, size_t *); +OSStatus SSLGetBufferedReadSize(SSLContextRef, size_t *); +OSStatus SSLRead(SSLContextRef, void *, size_t, size_t *); +OSStatus SSLWrite(SSLContextRef, const void *, size_t, size_t *); +OSStatus SSLClose(SSLContextRef); + +OSStatus SSLGetNumberSupportedCiphers(SSLContextRef, size_t *); +OSStatus SSLGetSupportedCiphers(SSLContextRef, SSLCipherSuite *, size_t *); +OSStatus SSLSetEnabledCiphers(SSLContextRef, const SSLCipherSuite *, size_t); +OSStatus SSLGetNumberEnabledCiphers(SSLContextRef, size_t *); +OSStatus SSLGetEnabledCiphers(SSLContextRef, SSLCipherSuite *, size_t *); +OSStatus SSLGetNegotiatedCipher(SSLContextRef, SSLCipherSuite *); +OSStatus SSLSetDiffieHellmanParams(SSLContextRef, const void *, size_t); +OSStatus SSLGetDiffieHellmanParams(SSLContextRef, const void **, size_t *); + +OSStatus SSLSetCertificateAuthorities(SSLContextRef, CFTypeRef, Boolean); +OSStatus SSLCopyCertificateAuthorities(SSLContextRef, CFArrayRef *); +OSStatus SSLCopyDistinguishedNames(SSLContextRef, CFArrayRef *); +OSStatus SSLSetCertificate(SSLContextRef, CFArrayRef); +OSStatus SSLGetClientCertificateState(SSLContextRef, + SSLClientCertificateState *); +OSStatus SSLCopyPeerTrust(SSLContextRef, SecTrustRef *trust); + +OSStatus SSLSetPeerDomainName(SSLContextRef, const char *, size_t); +OSStatus SSLGetPeerDomainNameLength(SSLContextRef, size_t *); +OSStatus SSLGetPeerDomainName(SSLContextRef, char *, size_t *); +""" + +MACROS = """ +""" + +CUSTOMIZATIONS = """ +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.c @@ -0,0 +1,22 @@ +// This file is dual licensed under the terms of the Apache License, Version +// 2.0, and the BSD License. See the LICENSE file in the root of this +// repository for complete details. + +uint8_t Cryptography_constant_time_bytes_eq(uint8_t *a, size_t len_a, + uint8_t *b, size_t len_b) { + size_t i = 0; + uint8_t mismatch = 0; + if (len_a != len_b) { + return 0; + } + for (i = 0; i < len_a; i++) { + mismatch |= a[i] ^ b[i]; + } + + /* Make sure any bits set are copied to the lowest bit */ + mismatch |= mismatch >> 4; + mismatch |= mismatch >> 2; + mismatch |= mismatch >> 1; + /* Now check the low bit to see if it's set */ + return (mismatch & 1) == 0; +} diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/constant_time.h @@ -0,0 +1,6 @@ +// This file is dual licensed under the terms of the Apache License, Version +// 2.0, and the BSD License. See the LICENSE file in the root of this +// repository for complete details. + +uint8_t Cryptography_constant_time_bytes_eq(uint8_t *, size_t, uint8_t *, + size_t); diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.c @@ -0,0 +1,65 @@ +// This file is dual licensed under the terms of the Apache License, Version +// 2.0, and the BSD License. See the LICENSE file in the root of this +// repository for complete details. + +/* Returns the value of the input with the most-significant-bit copied to all + of the bits. */ +static uint16_t Cryptography_DUPLICATE_MSB_TO_ALL(uint16_t a) { + return (1 - (a >> (sizeof(uint16_t) * 8 - 1))) - 1; +} + +/* This returns 0xFFFF if a < b else 0x0000, but does so in a constant time + fashion */ +static uint16_t Cryptography_constant_time_lt(uint16_t a, uint16_t b) { + a -= b; + return Cryptography_DUPLICATE_MSB_TO_ALL(a); +} + +uint8_t Cryptography_check_pkcs7_padding(const uint8_t *data, + uint16_t block_len) { + uint16_t i; + uint16_t pad_size = data[block_len - 1]; + uint16_t mismatch = 0; + for (i = 0; i < block_len; i++) { + unsigned int mask = Cryptography_constant_time_lt(i, pad_size); + uint16_t b = data[block_len - 1 - i]; + mismatch |= (mask & (pad_size ^ b)); + } + + /* Check to make sure the pad_size was within the valid range. */ + mismatch |= ~Cryptography_constant_time_lt(0, pad_size); + mismatch |= Cryptography_constant_time_lt(block_len, pad_size); + + /* Make sure any bits set are copied to the lowest bit */ + mismatch |= mismatch >> 8; + mismatch |= mismatch >> 4; + mismatch |= mismatch >> 2; + mismatch |= mismatch >> 1; + /* Now check the low bit to see if it's set */ + return (mismatch & 1) == 0; +} + +uint8_t Cryptography_check_ansix923_padding(const uint8_t *data, + uint16_t block_len) { + uint16_t i; + uint16_t pad_size = data[block_len - 1]; + uint16_t mismatch = 0; + /* Skip the first one with the pad size */ + for (i = 1; i < block_len; i++) { + unsigned int mask = Cryptography_constant_time_lt(i, pad_size); + uint16_t b = data[block_len - 1 - i]; + mismatch |= (mask & b); + } + + /* Check to make sure the pad_size was within the valid range. */ + mismatch |= ~Cryptography_constant_time_lt(0, pad_size); + mismatch |= Cryptography_constant_time_lt(block_len, pad_size); + + /* Make sure any bits set are copied to the lowest bit */ + mismatch |= mismatch >> 8; + mismatch |= mismatch >> 4; + mismatch |= mismatch >> 2; + mismatch |= mismatch >> 1; + /* Now check the low bit to see if it's set */ + return (mismatch & 1) == 0; +} diff --git a/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/hazmat_src/padding.h @@ -0,0 +1,6 @@ +// This file is dual licensed under the terms of the Apache License, Version +// 2.0, and the BSD License. See the LICENSE file in the root of this +// repository for complete details. + +uint8_t Cryptography_check_pkcs7_padding(const uint8_t *, uint8_t); +uint8_t Cryptography_check_ansix923_padding(const uint8_t *, uint8_t); diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/aes.py @@ -0,0 +1,50 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +static const int Cryptography_HAS_AES_WRAP; +static const int Cryptography_HAS_AES_CTR128_ENCRYPT; + +struct aes_key_st { + ...; +}; +typedef struct aes_key_st AES_KEY; +""" + +FUNCTIONS = """ +int AES_set_encrypt_key(const unsigned char *, const int, AES_KEY *); +int AES_set_decrypt_key(const unsigned char *, const int, AES_KEY *); + +int AES_wrap_key(AES_KEY *, const unsigned char *, unsigned char *, + const unsigned char *, unsigned int); +int AES_unwrap_key(AES_KEY *, const unsigned char *, unsigned char *, + const unsigned char *, unsigned int); +""" + +MACROS = """ +/* The ctr128_encrypt function is only useful in 1.0.0. We can use EVP for + this in 1.0.1+. */ +void AES_ctr128_encrypt(const unsigned char *, unsigned char *, + size_t, const AES_KEY *, unsigned char[], + unsigned char[], unsigned int *); +""" + +CUSTOMIZATIONS = """ +static const long Cryptography_HAS_AES_WRAP = 1; +#if CRYPTOGRAPHY_OPENSSL_110_OR_GREATER && !defined(LIBRESSL_VERSION_NUMBER) +static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 0; +void (*AES_ctr128_encrypt)(const unsigned char *, unsigned char *, + size_t, const AES_KEY *, + unsigned char[], unsigned char[], + unsigned int *) = NULL; +#else +static const int Cryptography_HAS_AES_CTR128_ENCRYPT = 1; +#endif +""" diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/asn1.py @@ -0,0 +1,164 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +INCLUDES = """ +#include +""" + +TYPES = """ +typedef int... time_t; + +typedef int ASN1_BOOLEAN; +typedef ... ASN1_INTEGER; + +struct asn1_string_st { + int length; + int type; + unsigned char *data; + long flags; +}; + +typedef struct asn1_string_st ASN1_OCTET_STRING; +typedef struct asn1_string_st ASN1_IA5STRING; +typedef struct asn1_string_st ASN1_BIT_STRING; +typedef struct asn1_string_st ASN1_TIME; +typedef ... ASN1_OBJECT; +typedef struct asn1_string_st ASN1_STRING; +typedef struct asn1_string_st ASN1_UTF8STRING; +typedef ... ASN1_TYPE; +typedef ... ASN1_GENERALIZEDTIME; +typedef ... ASN1_ENUMERATED; +typedef ... ASN1_ITEM; +typedef ... ASN1_VALUE; + +typedef ... ASN1_ITEM_EXP; + +typedef ... ASN1_UTCTIME; + +static const int V_ASN1_GENERALIZEDTIME; + +static const int MBSTRING_FLAG; +static const int MBSTRING_ASC; +static const int MBSTRING_BMP; +static const int MBSTRING_UTF8; From pypy.commits at gmail.com Tue Jun 4 12:51:13 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Jun 2019 09:51:13 -0700 (PDT) Subject: [pypy-commit] pypy optimizeopt-cleanup: revise optimizer docs Message-ID: <5cf6a181.1c69fb81.712d5.a890@mx.google.com> Author: Ronan Lamy Branch: optimizeopt-cleanup Changeset: r96737:33dbdadb7378 Date: 2019-06-04 17:50 +0100 http://bitbucket.org/pypy/pypy/changeset/33dbdadb7378/ Log: revise optimizer docs diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst --- a/rpython/doc/jit/optimizer.rst +++ b/rpython/doc/jit/optimizer.rst @@ -13,7 +13,7 @@ Before some optimizations are explained in more detail, it is essential to understand how traces look like. -The optimizer comes with a test suit. It contains many trace +The optimizer comes with a test suite. It contains many trace examples and you might want to take a look at it (in `rpython/jit/metainterp/optimizeopt/test/*.py`). The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. @@ -21,7 +21,7 @@ [p0,i0,i1] label(p0, i0, i1) - i2 = getarray_item_raw(p0, i0, descr=) + i2 = getarrayitem_raw(p0, i0, descr=) i3 = int_add(i1,i2) i4 = int_add(i0,1) i5 = int_le(i4, 100) # lower-or-equal @@ -32,7 +32,7 @@ to compare the Python code that constructed the trace:: from array import array - a = array('i',range(101)) + a = array('i', range(101)) sum = 0; i = 0 while i <= 100: # can be seen as label sum += a[i] @@ -131,20 +131,16 @@ Whenever such an operation is encountered (e.g. ``y = x & 0``), no operation is emitted. Instead the variable y is made equal to 0 -(= ``make_equal_to(op.result, 0)``). The variables found in a trace are -instances of Box classes that can be found in -`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again -and maps the boxes to the optimization values in the optimizer. When a -value is made equal, the two variable's boxes are made to point to the same -`OptValue` instance. +(= ``make_constant_int(op, 0)``). The variables found in a trace are instances +of classes that can be found in `rpython/jit/metainterp/history.py`. When a +value is made equal to another, its box is made to point to the other one. -**NOTE: this OptValue organization is currently being refactored in a branch.** Pure optimization ----------------- -Is interwoven into the basic optimizer. It saves operations, results, -arguments to be known to have pure semantics. +The 'pure' optimizations interwoven into the basic optimizer. It saves +operations, results, arguments to be known to have pure semantics. "Pure" here means the same as the ``jit.elidable`` decorator: free of "observable" side effects and referentially transparent From pypy.commits at gmail.com Tue Jun 4 15:25:35 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Jun 2019 12:25:35 -0700 (PDT) Subject: [pypy-commit] pypy optimizeopt-cleanup: Close branch optimizeopt-cleanup Message-ID: <5cf6c5af.1c69fb81.fad62.533a@mx.google.com> Author: Ronan Lamy Branch: optimizeopt-cleanup Changeset: r96738:decf9dbdb752 Date: 2019-06-04 19:25 +0000 http://bitbucket.org/pypy/pypy/changeset/decf9dbdb752/ Log: Close branch optimizeopt-cleanup From pypy.commits at gmail.com Tue Jun 4 15:25:55 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 04 Jun 2019 12:25:55 -0700 (PDT) Subject: [pypy-commit] pypy default: Merged in optimizeopt-cleanup (pull request #646) Message-ID: <5cf6c5c3.1c69fb81.b4f7f.07a7@mx.google.com> Author: Ronan Lamy Branch: Changeset: r96739:4ab755b28b85 Date: 2019-06-04 19:25 +0000 http://bitbucket.org/pypy/pypy/changeset/4ab755b28b85/ Log: Merged in optimizeopt-cleanup (pull request #646) Optimizeopt cleanup diff too long, truncating to 2000 out of 6785 lines diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst --- a/rpython/doc/jit/optimizer.rst +++ b/rpython/doc/jit/optimizer.rst @@ -13,7 +13,7 @@ Before some optimizations are explained in more detail, it is essential to understand how traces look like. -The optimizer comes with a test suit. It contains many trace +The optimizer comes with a test suite. It contains many trace examples and you might want to take a look at it (in `rpython/jit/metainterp/optimizeopt/test/*.py`). The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. @@ -21,7 +21,7 @@ [p0,i0,i1] label(p0, i0, i1) - i2 = getarray_item_raw(p0, i0, descr=) + i2 = getarrayitem_raw(p0, i0, descr=) i3 = int_add(i1,i2) i4 = int_add(i0,1) i5 = int_le(i4, 100) # lower-or-equal @@ -32,7 +32,7 @@ to compare the Python code that constructed the trace:: from array import array - a = array('i',range(101)) + a = array('i', range(101)) sum = 0; i = 0 while i <= 100: # can be seen as label sum += a[i] @@ -131,20 +131,16 @@ Whenever such an operation is encountered (e.g. ``y = x & 0``), no operation is emitted. Instead the variable y is made equal to 0 -(= ``make_equal_to(op.result, 0)``). The variables found in a trace are -instances of Box classes that can be found in -`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again -and maps the boxes to the optimization values in the optimizer. When a -value is made equal, the two variable's boxes are made to point to the same -`OptValue` instance. +(= ``make_constant_int(op, 0)``). The variables found in a trace are instances +of classes that can be found in `rpython/jit/metainterp/history.py`. When a +value is made equal to another, its box is made to point to the other one. -**NOTE: this OptValue organization is currently being refactored in a branch.** Pure optimization ----------------- -Is interwoven into the basic optimizer. It saves operations, results, -arguments to be known to have pure semantics. +The 'pure' optimizations interwoven into the basic optimizer. It saves +operations, results, arguments to be known to have pure semantics. "Pure" here means the same as the ``jit.elidable`` decorator: free of "observable" side effects and referentially transparent diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -3,23 +3,22 @@ from rpython.rtyper.annlowlevel import ( cast_instance_to_gcref, cast_gcref_to_instance) from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.debug import debug_start, debug_stop, debug_print, have_debug_prints +from rpython.rlib.debug import ( + debug_start, debug_stop, debug_print, have_debug_prints) from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib import rstack from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside from rpython.rlib.rjitlog import rjitlog as jl -from rpython.rlib.objectmodel import compute_unique_id -from rpython.conftest import option -from rpython.jit.metainterp.resoperation import ResOperation, rop,\ - get_deep_immutable_oplist, OpHelpers, InputArgInt, InputArgRef,\ - InputArgFloat -from rpython.jit.metainterp.history import (TreeLoop, Const, JitCellToken, +from rpython.jit.metainterp.resoperation import ( + ResOperation, rop, get_deep_immutable_oplist, OpHelpers, InputArgInt, + InputArgRef, InputArgFloat) +from rpython.jit.metainterp.history import (TreeLoop, JitCellToken, TargetToken, AbstractFailDescr, ConstInt) from rpython.jit.metainterp import history, jitexc from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.resume import (PENDINGFIELDSP, - ResumeDataDirectReader, AccumInfo) +from rpython.jit.metainterp.resume import ( + PENDINGFIELDSP, ResumeDataDirectReader) from rpython.jit.metainterp.resumecode import NUMBERING from rpython.jit.metainterp.support import adr2int from rpython.jit.codewriter import longlong @@ -30,6 +29,9 @@ raise SwitchToBlackhole(Counters.ABORT_BRIDGE) class CompileData(object): + """ An object that accumulates all of the necessary info for + the optimization phase, but does not actually have any other state + """ memo = None log_noopt = True @@ -37,10 +39,28 @@ for arg in self.trace.inputargs: arg.set_forwarded(None) -class LoopCompileData(CompileData): - """ An object that accumulates all of the necessary info for - the optimization phase, but does not actually have any other state + def optimize_trace(self, metainterp_sd, jitdriver_sd, memo): + """Optimize loop.operations to remove internal overheadish operations. + """ + from rpython.jit.metainterp.optimizeopt import build_opt_chain + # mark that a new trace has been started + log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE, metainterp_sd, None) + log.write_trace(self.trace) + if self.log_noopt: + metainterp_sd.logger_noopt.log_loop_from_trace(self.trace, memo=memo) + self.box_names_memo = memo + optimizations = build_opt_chain(self.enable_opts) + debug_start("jit-optimize") + try: + return self.optimize(metainterp_sd, jitdriver_sd, optimizations) + finally: + self.forget_optimization_info() + debug_stop("jit-optimize") + + +class PreambleCompileData(CompileData): + """ This is the case of label() ops label() """ def __init__(self, trace, runtime_boxes, call_pure_results=None, @@ -51,19 +71,12 @@ assert runtime_boxes is not None self.runtime_boxes = runtime_boxes - def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): - from rpython.jit.metainterp.optimizeopt.unroll import (UnrollOptimizer, - Optimizer) - - if unroll: - opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.optimize_preamble(self.trace, - self.runtime_boxes, - self.call_pure_results, - self.box_names_memo) - else: - opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.propagate_all_forward(self.trace, self.call_pure_results) + def optimize(self, metainterp_sd, jitdriver_sd, optimizations): + from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer + opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) + return opt.optimize_preamble( + self.trace, self.runtime_boxes, self.call_pure_results, + self.box_names_memo) class SimpleCompileData(CompileData): """ This represents label() ops jump with no extra info associated with @@ -76,26 +89,19 @@ self.call_pure_results = call_pure_results self.enable_opts = enable_opts - def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): + def optimize(self, metainterp_sd, jitdriver_sd, optimizations): from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer - from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge - - #assert not unroll opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations) - traceiter = self.trace.get_iter() - if self.resumestorage: - frontend_inputargs = self.trace.inputargs - deserialize_optimizer_knowledge(opt, self.resumestorage, - frontend_inputargs, - traceiter.inputargs) - return opt.propagate_all_forward(traceiter, self.call_pure_results) + return opt.optimize_loop( + self.trace, self.resumestorage, self.call_pure_results) class BridgeCompileData(CompileData): """ This represents ops() with a jump at the end that goes to some loop, we need to deal with virtual state and inlining of short preamble """ - def __init__(self, trace, runtime_boxes, resumestorage=None, call_pure_results=None, - enable_opts=None, inline_short_preamble=False): + def __init__(self, trace, runtime_boxes, resumestorage=None, + call_pure_results=None, enable_opts=None, + inline_short_preamble=False): self.trace = trace self.runtime_boxes = runtime_boxes self.call_pure_results = call_pure_results @@ -103,7 +109,7 @@ self.inline_short_preamble = inline_short_preamble self.resumestorage = resumestorage - def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): + def optimize(self, metainterp_sd, jitdriver_sd, optimizations): from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) @@ -115,29 +121,26 @@ class UnrolledLoopData(CompileData): """ This represents label() ops jump with extra info that's from the - run of LoopCompileData. Jump goes to the same label + run of PreambleCompileData. Jump goes to the same label """ log_noopt = False def __init__(self, trace, celltoken, state, - call_pure_results=None, enable_opts=None, - inline_short_preamble=True): + call_pure_results=None, enable_opts=None): self.trace = trace self.celltoken = celltoken self.enable_opts = enable_opts self.state = state self.call_pure_results = call_pure_results - self.inline_short_preamble = inline_short_preamble - def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): + def optimize(self, metainterp_sd, jitdriver_sd, optimizations): from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer - - assert unroll # we should not be here if it's disabled opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state, - self.call_pure_results, self.inline_short_preamble) + return opt.optimize_peeled_loop( + self.trace, self.celltoken, self.state, self.call_pure_results) def show_procedures(metainterp_sd, procedure=None, error=None): + from rpython.conftest import option # debugging if option and (option.view or option.viewloops): if error: @@ -173,19 +176,14 @@ original_jitcell_token = loop.original_jitcell_token assert original_jitcell_token is not None if metainterp_sd.warmrunnerdesc is not None: # for tests - assert original_jitcell_token.generation > 0 # has been registered with memmgr + assert original_jitcell_token.generation > 0 # has been registered with memmgr wref = weakref.ref(original_jitcell_token) clt = original_jitcell_token.compiled_loop_token clt.loop_token_wref = wref for op in loop.operations: descr = op.getdescr() - # not sure what descr.index is about if isinstance(descr, ResumeDescr): descr.rd_loop_token = clt # stick it there - #n = descr.index - #if n >= 0: # we also record the resumedescr number - # original_jitcell_token.compiled_loop_token.record_faildescr_index(n) - # pass if isinstance(descr, JitCellToken): # for a CALL_ASSEMBLER: record it as a potential jump. if descr is not original_jitcell_token: @@ -215,11 +213,8 @@ # ____________________________________________________________ - def compile_simple_loop(metainterp, greenkey, trace, runtime_args, enable_opts, cut_at): - from rpython.jit.metainterp.optimizeopt import optimize_trace - jitdriver_sd = metainterp.jitdriver_sd metainterp_sd = metainterp.staticdata jitcell_token = make_jitcell_token(jitdriver_sd) @@ -227,8 +222,8 @@ data = SimpleCompileData(trace, call_pure_results=call_pure_results, enable_opts=enable_opts) try: - loop_info, ops = optimize_trace(metainterp_sd, jitdriver_sd, - data, metainterp.box_names_memo) + loop_info, ops = data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: metainterp_sd.jitlog.trace_aborted() trace.cut_at(cut_at) @@ -253,12 +248,10 @@ return target_token def compile_loop(metainterp, greenkey, start, inputargs, jumpargs, - full_preamble_needed=True, try_disabling_unroll=False): + use_unroll=True): """Try to compile a new procedure by closing the current history back to the first operation. """ - from rpython.jit.metainterp.optimizeopt import optimize_trace - metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd history = metainterp.history @@ -269,28 +262,21 @@ faildescr=None, entry_bridge=False) # enable_opts = jitdriver_sd.warmstate.enable_opts - if try_disabling_unroll: - if 'unroll' not in enable_opts: - return None - enable_opts = enable_opts.copy() - del enable_opts['unroll'] - jitcell_token = make_jitcell_token(jitdriver_sd) cut_at = history.get_trace_position() history.record(rop.JUMP, jumpargs, None, descr=jitcell_token) if start != (0, 0, 0): trace = trace.cut_trace_from(start, inputargs) - if 'unroll' not in enable_opts or not metainterp.cpu.supports_guard_gc_type: + if not use_unroll: return compile_simple_loop(metainterp, greenkey, trace, jumpargs, enable_opts, cut_at) call_pure_results = metainterp.call_pure_results - preamble_data = LoopCompileData(trace, jumpargs, + preamble_data = PreambleCompileData(trace, jumpargs, call_pure_results=call_pure_results, enable_opts=enable_opts) try: - start_state, preamble_ops = optimize_trace(metainterp_sd, jitdriver_sd, - preamble_data, - metainterp.box_names_memo) + start_state, preamble_ops = preamble_data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: metainterp_sd.jitlog.trace_aborted() history.cut(cut_at) @@ -305,9 +291,8 @@ call_pure_results=call_pure_results, enable_opts=enable_opts) try: - loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd, - loop_data, - metainterp.box_names_memo) + loop_info, loop_ops = loop_data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: metainterp_sd.jitlog.trace_aborted() history.cut(cut_at) @@ -354,8 +339,6 @@ """Try to compile a new procedure by closing the current history back to the first operation. """ - from rpython.jit.metainterp.optimizeopt import optimize_trace - trace = metainterp.history.trace.cut_trace_from(start, inputargs) metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd @@ -375,29 +358,14 @@ call_pure_results=call_pure_results, enable_opts=enable_opts) try: - loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd, - loop_data, - metainterp.box_names_memo) + loop_info, loop_ops = loop_data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: - # Fall back on jumping directly to preamble + metainterp_sd.jitlog.trace_aborted() history.cut(cut) - history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token) - loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state, - call_pure_results=call_pure_results, - enable_opts=enable_opts, - inline_short_preamble=False) - try: - loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd, - loop_data, - metainterp.box_names_memo) - except InvalidLoop: - metainterp_sd.jitlog.trace_aborted() - history.cut(cut) - return None + return None label_op = loop_info.label_op - if label_op is None: - assert False, "unreachable code" # hint for some strange tests label_token = label_op.getdescr() assert isinstance(label_token, TargetToken) if label_token.short_preamble: @@ -405,9 +373,8 @@ label_token.short_preamble, metainterp.box_names_memo) loop = partial_trace loop.original_jitcell_token = loop_jitcell_token - loop.operations = (loop.operations + loop_info.extra_same_as + - [loop_info.label_op] - + loop_ops) + loop.operations = ( + loop.operations + loop_info.extra_same_as + [label_op] + loop_ops) quasi_immutable_deps = {} if loop_info.quasi_immutable_deps: @@ -419,8 +386,6 @@ target_token = loop.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, loop, inputargs) - - record_loop_or_bridge(metainterp_sd, loop) return target_token def get_box_replacement(op, allow_none=False): @@ -828,6 +793,7 @@ self, inputargs, new_loop.operations, new_loop.original_jitcell_token, metainterp.box_names_memo) + record_loop_or_bridge(metainterp.staticdata, new_loop) def make_a_counter_per_value(self, guard_value_op, index): assert guard_value_op.getopnum() == rop.GUARD_VALUE @@ -1038,18 +1004,16 @@ jitdriver_sd.warmstate.attach_procedure_to_interp( self.original_greenkey, jitcell_token) metainterp_sd.stats.add_jitcell_token(jitcell_token) + record_loop_or_bridge(metainterp_sd, new_loop) def get_resumestorage(self): return None -def compile_trace(metainterp, resumekey, runtime_boxes): +def compile_trace(metainterp, resumekey, runtime_boxes, ends_with_jump=False): """Try to compile a new bridge leading from the beginning of the history to some existing place. """ - - from rpython.jit.metainterp.optimizeopt import optimize_trace - # The history contains new operations to attach as the code for the # failure of 'resumekey.guard_op'. # @@ -1058,11 +1022,6 @@ metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd - # - jd_name = jitdriver_sd.jitdriver.name - metainterp_sd.jitlog.start_new_trace(metainterp_sd, - faildescr=resumekey, entry_bridge=False, jd_name=jd_name) - # if isinstance(resumekey, ResumeAtPositionDescr): inline_short_preamble = False else: @@ -1071,11 +1030,15 @@ trace = metainterp.history.trace jitdriver_sd = metainterp.jitdriver_sd enable_opts = jitdriver_sd.warmstate.enable_opts - call_pure_results = metainterp.call_pure_results resumestorage = resumekey.get_resumestorage() - if metainterp.history.ends_with_jump: + trace.tracing_done() + metainterp_sd.jitlog.start_new_trace(metainterp_sd, + faildescr=resumekey, entry_bridge=False, + jd_name=jitdriver_sd.jitdriver.name) + + if ends_with_jump: data = BridgeCompileData(trace, runtime_boxes, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts, @@ -1085,12 +1048,10 @@ call_pure_results=call_pure_results, enable_opts=enable_opts) try: - info, newops = optimize_trace(metainterp_sd, jitdriver_sd, - data, metainterp.box_names_memo) + info, newops = data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: metainterp_sd.jitlog.trace_aborted() - #pdb.post_mortem(sys.exc_info()[2]) - debug_print("compile_new_bridge: got an InvalidLoop") # XXX I am fairly convinced that optimize_bridge cannot actually raise # InvalidLoop debug_print('InvalidLoop in compile_new_bridge') @@ -1104,7 +1065,6 @@ new_trace.inputargs = info.inputargs target_token = new_trace.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, new_trace, inputargs) - record_loop_or_bridge(metainterp_sd, new_trace) return target_token new_trace.inputargs = info.renamed_inputargs metainterp.retrace_needed(new_trace, info) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -689,7 +689,6 @@ class History(object): - ends_with_jump = False trace = None def __init__(self): diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py --- a/rpython/jit/metainterp/opencoder.py +++ b/rpython/jit/metainterp/opencoder.py @@ -7,12 +7,13 @@ """ from rpython.jit.metainterp.history import ( - ConstInt, Const, ConstFloat, ConstPtr, new_ref_dict) + ConstInt, Const, ConstFloat, ConstPtr, new_ref_dict, SwitchToBlackhole) from rpython.jit.metainterp.resoperation import AbstractResOp, AbstractInputArg,\ ResOperation, oparity, rop, opwithdescr, GuardResOp, IntOp, FloatOp, RefOp,\ opclasses from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib.objectmodel import we_are_translated, specialize +from rpython.rlib.jit import Counters from rpython.rtyper.lltypesystem import rffi, lltype, llmemory TAGINT, TAGCONSTPTR, TAGCONSTOTHER, TAGBOX = range(4) @@ -301,7 +302,8 @@ def tracing_done(self): from rpython.rlib.debug import debug_start, debug_stop, debug_print - assert not self.tag_overflow + if self.tag_overflow: + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) self._bigints_dict = {} self._refs_dict = new_ref_dict() diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py --- a/rpython/jit/metainterp/optimizeopt/__init__.py +++ b/rpython/jit/metainterp/optimizeopt/__init__.py @@ -1,4 +1,3 @@ -from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer from rpython.jit.metainterp.optimizeopt.rewrite import OptRewrite from rpython.jit.metainterp.optimizeopt.intbounds import OptIntBounds from rpython.jit.metainterp.optimizeopt.virtualize import OptVirtualize @@ -7,10 +6,8 @@ from rpython.jit.metainterp.optimizeopt.simplify import OptSimplify from rpython.jit.metainterp.optimizeopt.pure import OptPure from rpython.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce -from rpython.rlib.rjitlog import rjitlog as jl from rpython.rlib.jit import PARAMETERS, ENABLE_ALL_OPTS from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib.debug import debug_start, debug_stop, debug_print ALL_OPTS = [('intbounds', OptIntBounds), @@ -31,45 +28,17 @@ assert ENABLE_ALL_OPTS == ALL_OPTS_NAMES, ( 'please fix rlib/jit.py to say ENABLE_ALL_OPTS = %r' % (ALL_OPTS_NAMES,)) -def build_opt_chain(metainterp_sd, enable_opts): +def build_opt_chain(enable_opts): optimizations = [] - unroll = 'unroll' in enable_opts # 'enable_opts' is normally a dict - if (metainterp_sd.cpu is not None and - not metainterp_sd.cpu.supports_guard_gc_type): - unroll = False for name, opt in unroll_all_opts: if name in enable_opts: if opt is not None: o = opt() optimizations.append(o) - - if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts - or 'heap' not in enable_opts or 'pure' not in enable_opts): - optimizations.append(OptSimplify(unroll)) - - return optimizations, unroll - -def optimize_trace(metainterp_sd, jitdriver_sd, compile_data, memo=None): - """Optimize loop.operations to remove internal overheadish operations. - """ - debug_start("jit-optimize") - try: - # mark that a new trace has been started - log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE, metainterp_sd, None) - log.write_trace(compile_data.trace) - if compile_data.log_noopt: - metainterp_sd.logger_noopt.log_loop_from_trace(compile_data.trace, memo=memo) - if memo is None: - memo = {} - compile_data.box_names_memo = memo - optimizations, unroll = build_opt_chain(metainterp_sd, - compile_data.enable_opts) - return compile_data.optimize(metainterp_sd, jitdriver_sd, - optimizations, unroll) - finally: - compile_data.forget_optimization_info() - debug_stop("jit-optimize") + if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts or + 'heap' not in enable_opts or 'pure' not in enable_opts): + optimizations.append(OptSimplify()) + return optimizations if __name__ == '__main__': print ALL_OPTS_NAMES - diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -3,6 +3,7 @@ from rpython.jit.metainterp import resumecode from rpython.jit.metainterp.history import Const, ConstInt, CONST_NULL +from .info import getptrinfo # adds the following sections at the end of the resume code: @@ -75,7 +76,7 @@ for box in liveboxes: if box is None or box.type != "r": continue - info = optimizer.getptrinfo(box) + info = getptrinfo(box) known_class = info is not None and info.get_known_class(optimizer.cpu) is not None bitfield <<= 1 bitfield |= known_class diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -1,26 +1,19 @@ -import os from collections import OrderedDict from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.metainterp.optimizeopt.util import args_dict -from rpython.jit.metainterp.history import Const, ConstInt, new_ref_dict -from rpython.jit.metainterp.jitexc import JitException +from rpython.jit.metainterp.history import new_ref_dict from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED -from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method +from rpython.jit.metainterp.optimizeopt.util import ( + make_dispatcher_method, get_box_replacement) from rpython.jit.metainterp.optimizeopt.intutils import IntBound from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\ - GuardResOp +from rpython.jit.metainterp.resoperation import rop from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.optimizeopt import info - -class BogusImmutableField(JitException): - pass - - class AbstractCachedEntry(object): """ abstract base class abstracting over the difference between caching struct fields and array items. """ @@ -56,40 +49,32 @@ descr, index=-1): assert self._lazy_set is None for i, info in enumerate(self.cached_infos): - structbox = optimizer.get_box_replacement(self.cached_structs[i]) + structbox = get_box_replacement(self.cached_structs[i]) info.produce_short_preamble_ops(structbox, descr, index, optimizer, shortboxes) - def possible_aliasing(self, optheap, opinfo): + def possible_aliasing(self, opinfo): # If lazy_set is set and contains a setfield on a different # structvalue, then we are annoyed, because it may point to either # the same or a different structure at runtime. # XXX constants? return (self._lazy_set is not None - and (not optheap.getptrinfo( - self._lazy_set.getarg(0)).same_info(opinfo))) + and not info.getptrinfo(self._lazy_set.getarg(0)).same_info(opinfo)) def do_setfield(self, optheap, op): # Update the state with the SETFIELD_GC/SETARRAYITEM_GC operation 'op'. structinfo = optheap.ensure_ptr_info_arg0(op) - arg1 = optheap.get_box_replacement(self._get_rhs_from_set_op(op)) - if self.possible_aliasing(optheap, structinfo): + arg1 = get_box_replacement(self._get_rhs_from_set_op(op)) + if self.possible_aliasing(structinfo): self.force_lazy_set(optheap, op.getdescr()) - assert not self.possible_aliasing(optheap, structinfo) + assert not self.possible_aliasing(structinfo) cached_field = self._getfield(structinfo, op.getdescr(), optheap, False) if cached_field is not None: - cached_field = optheap.get_box_replacement(cached_field) - - # Hack to ensure constants are imported from the preamble - # XXX no longer necessary? - #if cached_fieldvalue and fieldvalue.is_constant(): - # optheap.optimizer.ensure_imported(cached_fieldvalue) - # cached_fieldvalue = self._cached_fields.get(structvalue, None) + cached_field = cached_field.get_box_replacement() if not cached_field or not cached_field.same_box(arg1): # common case: store the 'op' as lazy_set self._lazy_set = op - else: # this is the case where the pending setfield ends up # storing precisely the value that is already there, @@ -104,11 +89,11 @@ def getfield_from_cache(self, optheap, opinfo, descr): # Returns the up-to-date field's value, or None if not cached. - if self.possible_aliasing(optheap, opinfo): + if self.possible_aliasing(opinfo): self.force_lazy_set(optheap, descr) if self._lazy_set is not None: op = self._lazy_set - return optheap.get_box_replacement(self._get_rhs_from_set_op(op)) + return get_box_replacement(self._get_rhs_from_set_op(op)) else: res = self._getfield(opinfo, descr, optheap) if res is not None: @@ -140,7 +125,6 @@ elif not can_cache: self.invalidate(descr) - # abstract methods def _get_rhs_from_set_op(self, op): @@ -167,8 +151,8 @@ return op.getarg(1) def put_field_back_to_info(self, op, opinfo, optheap): - arg = optheap.get_box_replacement(op.getarg(1)) - struct = optheap.get_box_replacement(op.getarg(0)) + arg = get_box_replacement(op.getarg(1)) + struct = get_box_replacement(op.getarg(0)) opinfo.setfield(op.getdescr(), struct, arg, optheap=optheap, cf=self) def _getfield(self, opinfo, descr, optheap, true_force=True): @@ -216,8 +200,8 @@ return res def put_field_back_to_info(self, op, opinfo, optheap): - arg = optheap.get_box_replacement(op.getarg(2)) - struct = optheap.get_box_replacement(op.getarg(0)) + arg = get_box_replacement(op.getarg(2)) + struct = get_box_replacement(op.getarg(0)) opinfo.setitem(op.getdescr(), self.index, struct, arg, optheap=optheap, cf=self) def invalidate(self, descr): @@ -412,8 +396,8 @@ d = self.cached_dict_reads[descr1] = args_dict() self.corresponding_array_descrs[descrs[1]] = descr1 # - key = [self.optimizer.get_box_replacement(op.getarg(1)), # dict - self.optimizer.get_box_replacement(op.getarg(2))] # key + key = [get_box_replacement(op.getarg(1)), # dict + get_box_replacement(op.getarg(2))] # key # other args can be ignored here (hash, store_flag) try: res_v = d[key] @@ -525,7 +509,7 @@ # guards' resume data is that of a virtual object that is stored # into a field of a non-virtual object. Here, 'op' in either # SETFIELD_GC or SETARRAYITEM_GC. - opinfo = self.getptrinfo(op.getarg(0)) + opinfo = info.getptrinfo(op.getarg(0)) assert not opinfo.is_virtual() # it must be a non-virtual if self.optimizer.is_virtual(op.getarg(2)): pendingfields.append(op) @@ -564,13 +548,6 @@ def optimize_SETFIELD_GC(self, op): self.setfield(op) - #opnum = OpHelpers.getfield_pure_for_descr(op.getdescr()) - #if self.has_pure_result(opnum, [op.getarg(0)], - # op.getdescr()): - # os.write(2, '[bogus _immutable_field_ declaration: %s]\n' % - # (op.getdescr().repr_of_descr())) - # raise BogusImmutableField - # def setfield(self, op): cf = self.field_cache(op.getdescr()) @@ -606,8 +583,8 @@ index = indexb.getint() cf = self.arrayitem_cache(op.getdescr(), index) arrayinfo.setitem(op.getdescr(), indexb.getint(), - self.get_box_replacement(op.getarg(0)), - self.get_box_replacement(op), optheap=self, + get_box_replacement(op.getarg(0)), + get_box_replacement(op), optheap=self, cf=cf) optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I @@ -639,13 +616,6 @@ optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_PURE_I def optimize_SETARRAYITEM_GC(self, op): - #opnum = OpHelpers.getarrayitem_pure_for_descr(op.getdescr()) - #if self.has_pure_result(opnum, [op.getarg(0), op.getarg(1)], - # op.getdescr()): - # os.write(2, '[bogus immutable array declaration: %s]\n' % - # (op.getdescr().repr_of_descr())) - # raise BogusImmutableField - # indexb = self.getintbound(op.getarg(1)) if indexb.is_constant(): arrayinfo = self.ensure_ptr_info_arg0(op) @@ -680,8 +650,7 @@ # check that the value is still correct; it could have changed # already between the tracing and now. In this case, we mark the loop # as invalid - if not qmutdescr.is_still_valid_for( - self.get_box_replacement(op.getarg(0))): + if not qmutdescr.is_still_valid_for(get_box_replacement(op.getarg(0))): raise InvalidLoop('quasi immutable field changed during tracing') # record as an out-of-line guard if self.optimizer.quasi_immutable_deps is None: @@ -701,10 +670,10 @@ result_getfield = [] for descr, cf in self.cached_fields.iteritems(): if cf._lazy_set: - continue # XXX safe default for now + continue # XXX safe default for now parent_descr = descr.get_parent_descr() if not parent_descr.is_object(): - continue # XXX could be extended to non-instance objects + continue # XXX could be extended to non-instance objects for i, box1 in enumerate(cf.cached_structs): if not box1.is_constant() and box1 not in available_boxes: continue @@ -721,7 +690,7 @@ for descr, indexdict in self.cached_arrayitems.iteritems(): for index, cf in indexdict.iteritems(): if cf._lazy_set: - continue # XXX safe default for now + continue # XXX safe default for now for i, box1 in enumerate(cf.cached_structs): if not box1.is_constant() and box1 not in available_boxes: continue diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -1,13 +1,14 @@ -from rpython.rlib.objectmodel import specialize, we_are_translated, compute_hash -from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\ - rop, OpHelpers -from rpython.jit.metainterp.history import ConstInt, Const -from rpython.rtyper.lltypesystem import lltype, llmemory -from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation +from rpython.rlib.objectmodel import specialize, compute_hash +from rpython.jit.metainterp.resoperation import ( + AbstractValue, ResOperation, rop, OpHelpers) +from rpython.jit.metainterp.history import ConstInt, ConstPtr, Const +from rpython.rtyper.lltypesystem import lltype +from rpython.jit.metainterp.optimizeopt.rawbuffer import ( + RawBuffer, InvalidRawOperation) from rpython.jit.metainterp.executor import execute from rpython.jit.metainterp.optimize import InvalidLoop - +from .util import get_box_replacement INFO_NULL = 0 INFO_NONNULL = 1 @@ -51,7 +52,7 @@ def force_at_the_end_of_preamble(self, op, optforce, rec): if not self.is_virtual(): - return optforce.get_box_replacement(op) + return get_box_replacement(op) return self._force_at_the_end_of_preamble(op, optforce, rec) def get_known_class(self, cpu): @@ -84,7 +85,7 @@ @specialize.arg(2) def get_constant_string_spec(self, string_optimizer, mode): - return None # can't be constant + return None # can't be constant class NonNullPtrInfo(PtrInfo): _attrs_ = ('last_guard_pos',) @@ -109,7 +110,7 @@ def mark_last_guard(self, optimizer): if (optimizer.getlastop() is None or - not optimizer.getlastop().is_guard()): + not optimizer.getlastop().is_guard()): # there can be a really emitted operation that's not a guard # e.g. a setfield, ignore those return @@ -140,12 +141,12 @@ constptr = optforce.optimizer.constant_fold(op) op.set_forwarded(constptr) self._is_virtual = False - self._force_elements_immutable(self.descr, constptr, optforce) + self._force_elements_immutable(self.descr, constptr, optforce.optimizer) return constptr # op.set_forwarded(None) optforce.emit_extra(op) - newop = optforce.getlastop() + newop = optforce.optimizer.getlastop() if newop is not op: op.set_forwarded(newop) newop.set_forwarded(self) @@ -161,14 +162,14 @@ def is_virtual(self): return self._is_virtual - def _visitor_walk_recursive(self, op, visitor, optimizer): + def _visitor_walk_recursive(self, op, visitor): raise NotImplementedError("abstract") - def visitor_walk_recursive(self, instbox, visitor, optimizer): + def visitor_walk_recursive(self, instbox, visitor): instbox = instbox.get_box_replacement() if visitor.already_seen_virtual(instbox): return - return self._visitor_walk_recursive(instbox, visitor, optimizer) + return self._visitor_walk_recursive(instbox, visitor) class AbstractStructPtrInfo(AbstractVirtualPtrInfo): @@ -181,7 +182,7 @@ self.descr = descr self._fields = [None] * len(descr.get_all_fielddescrs()) if index >= len(self._fields): - self.descr = descr # a more precise descr + self.descr = descr # a more precise descr # we found out a subclass with more fields extra_len = len(descr.get_all_fielddescrs()) - len(self._fields) self._fields = self._fields + [None] * extra_len @@ -218,7 +219,7 @@ for i, fielddescr in enumerate(descr.get_all_fielddescrs()): fld = self._fields[i] if fld is not None: - subbox = optforce.force_box(fld) + subbox = optforce.optimizer.force_box(fld) setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], descr=fielddescr) self._fields[i] = None @@ -226,31 +227,30 @@ def _force_at_the_end_of_preamble(self, op, optforce, rec): if self._fields is None: - return optforce.get_box_replacement(op) + return get_box_replacement(op) if self in rec: - return optforce.get_box_replacement(op) + return get_box_replacement(op) rec[self] = None for i, fldbox in enumerate(self._fields): if fldbox is not None: - info = optforce.getptrinfo(fldbox) + info = getptrinfo(fldbox) if info is not None: fldbox = info.force_at_the_end_of_preamble(fldbox, optforce, rec) self._fields[i] = fldbox return op - def _visitor_walk_recursive(self, instbox, visitor, optimizer): + def _visitor_walk_recursive(self, instbox, visitor): lst = self.descr.get_all_fielddescrs() assert self.is_virtual() - visitor.register_virtual_fields(instbox, - [optimizer.get_box_replacement(box) - for box in self._fields]) + visitor.register_virtual_fields( + instbox, [get_box_replacement(box) for box in self._fields]) for i in range(len(lst)): op = self._fields[i] if op: - fieldinfo = optimizer.getptrinfo(op) + fieldinfo = getptrinfo(op) if fieldinfo and fieldinfo.is_virtual(): - fieldinfo.visitor_walk_recursive(op, visitor, optimizer) + fieldinfo.visitor_walk_recursive(op, visitor) def produce_short_preamble_ops(self, structbox, fielddescr, index, optimizer, shortboxes): @@ -259,7 +259,7 @@ if fielddescr.get_index() >= len(self._fields): # we don't know about this item return - op = optimizer.get_box_replacement(self._fields[fielddescr.get_index()]) + op = get_box_replacement(self._fields[fielddescr.get_index()]) if op is None: # XXX same bug as in serialize_opt: # op should never be None, because that's an invariant violation in @@ -290,7 +290,7 @@ if op.is_constant(): pass # it is a constant value: ok else: - fieldinfo = optimizer.getptrinfo(op) + fieldinfo = getptrinfo(op) if fieldinfo and fieldinfo.is_virtual(): # recursive check if memo is None: @@ -302,12 +302,12 @@ return False # not a constant at all return True - def _force_elements_immutable(self, descr, constptr, optforce): + def _force_elements_immutable(self, descr, constptr, optimizer): for i, fielddescr in enumerate(descr.get_all_fielddescrs()): fld = self._fields[i] - subbox = optforce.force_box(fld) + subbox = optimizer.force_box(fld) assert isinstance(subbox, Const) - execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC, + execute(optimizer.cpu, None, rop.SETFIELD_GC, fielddescr, constptr, subbox) class InstancePtrInfo(AbstractStructPtrInfo): @@ -435,8 +435,8 @@ [op, ConstInt(offset), itembox], descr=descr) optforce.emit_extra(setfield_op) - def _visitor_walk_recursive(self, op, visitor, optimizer): - itemboxes = [optimizer.get_box_replacement(box) + def _visitor_walk_recursive(self, op, visitor): + itemboxes = [get_box_replacement(box) for box in self._get_buffer().values] visitor.register_virtual_fields(op, itemboxes) # there can be no virtuals stored in raw buffer @@ -465,21 +465,21 @@ return self.parent is not None def getitem_raw(self, offset, itemsize, descr): - return self.parent.getitem_raw(self.offset+offset, itemsize, descr) + return self.parent.getitem_raw(self.offset + offset, itemsize, descr) def setitem_raw(self, offset, itemsize, descr, itemop): - self.parent.setitem_raw(self.offset+offset, itemsize, descr, itemop) + self.parent.setitem_raw(self.offset + offset, itemsize, descr, itemop) def _force_elements(self, op, optforce, descr): if self.parent.is_virtual(): self.parent._force_elements(op, optforce, descr) self.parent = None - def _visitor_walk_recursive(self, op, visitor, optimizer): - source_op = optimizer.get_box_replacement(op.getarg(0)) + def _visitor_walk_recursive(self, op, visitor): + source_op = get_box_replacement(op.getarg(0)) visitor.register_virtual_fields(op, [source_op]) if self.parent.is_virtual(): - self.parent.visitor_walk_recursive(source_op, visitor, optimizer) + self.parent.visitor_walk_recursive(source_op, visitor) @specialize.argtype(1) def visitor_dispatch_virtual_type(self, visitor): @@ -534,7 +534,7 @@ def _force_elements(self, op, optforce, descr): # XXX descr = op.getdescr() - const = optforce.new_const_item(self.descr) + const = optforce.optimizer.new_const_item(self.descr) for i in range(self.length): item = self._items[i] if item is None: @@ -543,7 +543,7 @@ # clear the item so we don't know what's there self._items[i] = None continue - subbox = optforce.force_box(item) + subbox = optforce.optimizer.force_box(item) setop = ResOperation(rop.SETARRAYITEM_GC, [op, ConstInt(i), subbox], descr=descr) @@ -570,17 +570,15 @@ def getlength(self): return self.length - def _visitor_walk_recursive(self, instbox, visitor, optimizer): - itemops = [optimizer.get_box_replacement(item) - for item in self._items] + def _visitor_walk_recursive(self, instbox, visitor): + itemops = [get_box_replacement(item) for item in self._items] visitor.register_virtual_fields(instbox, itemops) for i in range(self.getlength()): itemop = self._items[i] - if (itemop is not None and - not isinstance(itemop, Const)): - ptrinfo = optimizer.getptrinfo(itemop) + if (itemop is not None and not isinstance(itemop, Const)): + ptrinfo = getptrinfo(itemop) if ptrinfo and ptrinfo.is_virtual(): - ptrinfo.visitor_walk_recursive(itemop, visitor, optimizer) + ptrinfo.visitor_walk_recursive(itemop, visitor) @specialize.argtype(1) def visitor_dispatch_virtual_type(self, visitor): @@ -596,7 +594,7 @@ item = self._items[index] if item is not None: # see comment in AbstractStructPtrInfo.produce_short_preamble_ops - op = optimizer.get_box_replacement(item) + op = get_box_replacement(item) opnum = OpHelpers.getarrayitem_for_descr(descr) getarrayitem_op = ResOperation(opnum, [structbox, ConstInt(index)], descr=descr) @@ -604,13 +602,13 @@ def _force_at_the_end_of_preamble(self, op, optforce, rec): if self._items is None: - return optforce.get_box_replacement(op) + return get_box_replacement(op) if self in rec: - return optforce.get_box_replacement(op) + return get_box_replacement(op) rec[self] = None for i, fldbox in enumerate(self._items): if fldbox is not None: - info = optforce.getptrinfo(fldbox) + info = getptrinfo(fldbox) if info is not None: fldbox = info.force_at_the_end_of_preamble(fldbox, optforce, rec) @@ -640,7 +638,7 @@ def _compute_index(self, index, fielddescr): all_fdescrs = fielddescr.get_arraydescr().get_all_fielddescrs() if all_fdescrs is None: - return 0 # annotation hack + return 0 # annotation hack one_size = len(all_fdescrs) return index * one_size + fielddescr.get_field_descr().get_index() @@ -659,7 +657,7 @@ for fielddescr in fielddescrs: fld = self._items[i] if fld is not None: - subbox = optforce.force_box(fld) + subbox = optforce.optimizer.force_box(fld) setfieldop = ResOperation(rop.SETINTERIORFIELD_GC, [op, ConstInt(index), subbox], descr=fielddescr) @@ -668,21 +666,18 @@ # if it does, we would need a fix here i += 1 - def _visitor_walk_recursive(self, instbox, visitor, optimizer): - itemops = [optimizer.get_box_replacement(item) - for item in self._items] + def _visitor_walk_recursive(self, instbox, visitor): + itemops = [get_box_replacement(item) for item in self._items] visitor.register_virtual_fields(instbox, itemops) fielddescrs = self.descr.get_all_fielddescrs() i = 0 for index in range(self.getlength()): for fielddescr in fielddescrs: itemop = self._items[i] - if (itemop is not None and - not isinstance(itemop, Const)): - ptrinfo = optimizer.getptrinfo(itemop) + if (itemop is not None and not isinstance(itemop, Const)): + ptrinfo = getptrinfo(itemop) if ptrinfo and ptrinfo.is_virtual(): - ptrinfo.visitor_walk_recursive(itemop, visitor, - optimizer) + ptrinfo.visitor_walk_recursive(itemop, visitor) i += 1 @specialize.argtype(1) @@ -705,7 +700,8 @@ def _get_info(self, descr, optheap): ref = self._const.getref_base() - if not ref: raise InvalidLoop # null protection + if not ref: + raise InvalidLoop # null protection info = optheap.const_infos.get(ref, None) if info is None: info = StructPtrInfo(descr) @@ -714,7 +710,8 @@ def _get_array_info(self, descr, optheap): ref = self._const.getref_base() - if not ref: raise InvalidLoop # null protection + if not ref: + raise InvalidLoop # null protection info = optheap.const_infos.get(ref, None) if info is None: info = ArrayPtrInfo(descr) @@ -780,8 +777,8 @@ return self._unpack_str(mode) def getlenbound(self, mode): - from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound,\ - IntLowerBound + from rpython.jit.metainterp.optimizeopt.intutils import ( + ConstIntBound, IntLowerBound) length = self.getstrlen1(mode) if length < 0: @@ -848,3 +845,36 @@ def make_guards(self, op, short, optimizer): short.append(ResOperation(rop.GUARD_VALUE, [op, self._const])) + + +def getrawptrinfo(op): + from rpython.jit.metainterp.optimizeopt.intutils import IntBound + assert op.type == 'i' + op = op.get_box_replacement() + assert op.type == 'i' + if isinstance(op, ConstInt): + return ConstPtrInfo(op) + fw = op.get_forwarded() + if isinstance(fw, IntBound): + return None + if fw is not None: + assert isinstance(fw, AbstractRawPtrInfo) + return fw + return None + +def getptrinfo(op): + if op.type == 'i': + return getrawptrinfo(op) + elif op.type == 'f': + return None + assert op.type == 'r' + op = get_box_replacement(op) + assert op.type == 'r' + if isinstance(op, ConstPtr): + return ConstPtrInfo(op) + fw = op.get_forwarded() + if fw is not None: + assert isinstance(fw, PtrInfo) + return fw + return None + diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -5,7 +5,9 @@ IntLowerBound, IntUpperBound, ConstIntBound) from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1, CONST_0) -from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method +from rpython.jit.metainterp.optimizeopt.util import ( + make_dispatcher_method, get_box_replacement) +from .info import getptrinfo from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.optimizeopt import vstring from rpython.jit.codewriter.effectinfo import EffectInfo @@ -64,8 +66,8 @@ postprocess_GUARD_VALUE = _postprocess_guard_true_false_value def optimize_INT_OR_or_XOR(self, op): - v1 = self.get_box_replacement(op.getarg(0)) - v2 = self.get_box_replacement(op.getarg(1)) + v1 = get_box_replacement(op.getarg(0)) + v2 = get_box_replacement(op.getarg(1)) if v1 is v2: if op.getopnum() == rop.INT_OR: self.make_equal_to(op, v1) @@ -75,9 +77,9 @@ return self.emit(op) def postprocess_INT_OR_or_XOR(self, op): - v1 = self.get_box_replacement(op.getarg(0)) + v1 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(v1) - v2 = self.get_box_replacement(op.getarg(1)) + v2 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(v2) b = b1.or_bound(b2) self.getintbound(op).intersect(b) @@ -108,8 +110,8 @@ self.getintbound(op).intersect(b) def optimize_INT_ADD(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) if self.is_raw_ptr(arg1) or self.is_raw_ptr(arg2): return self.emit(op) v1 = self.getintbound(arg1) @@ -130,8 +132,8 @@ arg1 = self.optimizer.as_operation(arg1) if arg1 is not None: if arg1.getopnum() == rop.INT_ADD: - prod_arg1 = self.get_box_replacement(arg1.getarg(0)) - prod_arg2 = self.get_box_replacement(arg1.getarg(1)) + prod_arg1 = get_box_replacement(arg1.getarg(0)) + prod_arg2 = get_box_replacement(arg1.getarg(1)) prod_v1 = self.getintbound(prod_arg1) prod_v2 = self.getintbound(prod_arg2) @@ -196,9 +198,9 @@ return self.emit(op) def postprocess_INT_LSHIFT(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) + arg0 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(arg0) - arg1 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(arg1) r = self.getintbound(op) b = b1.lshift_bound(b2) @@ -290,8 +292,8 @@ r.intersect(resbound) def optimize_INT_SUB_OVF(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) - arg1 = self.get_box_replacement(op.getarg(1)) + arg0 = get_box_replacement(op.getarg(0)) + arg1 = get_box_replacement(op.getarg(1)) b0 = self.getintbound(arg0) b1 = self.getintbound(arg1) if arg0.same_box(arg1): @@ -304,8 +306,8 @@ return self.emit(op) def postprocess_INT_SUB_OVF(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) - arg1 = self.get_box_replacement(op.getarg(1)) + arg0 = get_box_replacement(op.getarg(0)) + arg1 = get_box_replacement(op.getarg(1)) b0 = self.getintbound(arg0) b1 = self.getintbound(arg1) resbound = b0.sub_bound(b1) @@ -329,8 +331,8 @@ r.intersect(resbound) def optimize_INT_LT(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b1.known_lt(b2): @@ -341,8 +343,8 @@ return self.emit(op) def optimize_INT_GT(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b1.known_gt(b2): @@ -353,8 +355,8 @@ return self.emit(op) def optimize_INT_LE(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b1.known_le(b2) or arg1 is arg2: @@ -365,8 +367,8 @@ return self.emit(op) def optimize_INT_GE(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b1.known_ge(b2) or arg1 is arg2: @@ -377,9 +379,9 @@ return self.emit(op) def optimize_INT_EQ(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) + arg0 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(arg0) - arg1 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(arg1) if b1.known_gt(b2): self.make_constant_int(op, 0) @@ -391,9 +393,9 @@ return self.emit(op) def optimize_INT_NE(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) + arg0 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(arg0) - arg1 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(arg1) if b1.known_gt(b2): self.make_constant_int(op, 1) @@ -442,7 +444,7 @@ def postprocess_STRLEN(self, op): self.make_nonnull_str(op.getarg(0), vstring.mode_string) - array = self.getptrinfo(op.getarg(0)) + array = getptrinfo(op.getarg(0)) self.optimizer.setintbound(op, array.getlenbound(vstring.mode_string)) def optimize_UNICODELEN(self, op): @@ -450,7 +452,7 @@ def postprocess_UNICODELEN(self, op): self.make_nonnull_str(op.getarg(0), vstring.mode_unicode) - array = self.getptrinfo(op.getarg(0)) + array = getptrinfo(op.getarg(0)) self.optimizer.setintbound(op, array.getlenbound(vstring.mode_unicode)) def optimize_STRGETITEM(self, op): @@ -458,7 +460,7 @@ def postprocess_STRGETITEM(self, op): v1 = self.getintbound(op) - v2 = self.getptrinfo(op.getarg(0)) + v2 = getptrinfo(op.getarg(0)) intbound = self.getintbound(op.getarg(1)) if (intbound.has_lower and v2 is not None and v2.getlenbound(vstring.mode_string) is not None): @@ -523,7 +525,7 @@ def postprocess_UNICODEGETITEM(self, op): b1 = self.getintbound(op) b1.make_ge(IntLowerBound(0)) - v2 = self.getptrinfo(op.getarg(0)) + v2 = getptrinfo(op.getarg(0)) intbound = self.getintbound(op.getarg(1)) if (intbound.has_lower and v2 is not None and v2.getlenbound(vstring.mode_unicode) is not None): diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -1,16 +1,19 @@ from rpython.jit.metainterp import jitprof, resume, compile from rpython.jit.metainterp.executor import execute_nonspec_const from rpython.jit.metainterp.history import ( - Const, ConstInt, ConstPtr, CONST_NULL, new_ref_dict) -from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\ - ConstIntBound, MININT, MAXINT, IntUnbounded -from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, AbstractResOp, GuardResOp,\ - OpHelpers + Const, ConstInt, CONST_NULL, new_ref_dict) +from rpython.jit.metainterp.optimizeopt.intutils import ( + IntBound, ConstIntBound, MININT, MAXINT, IntUnbounded) +from rpython.jit.metainterp.optimizeopt.util import ( + make_dispatcher_method, get_box_replacement) +from rpython.jit.metainterp.optimizeopt.bridgeopt import ( + deserialize_optimizer_knowledge) +from rpython.jit.metainterp.resoperation import ( + rop, AbstractResOp, GuardResOp, OpHelpers) +from .info import getrawptrinfo, getptrinfo from rpython.jit.metainterp.optimizeopt import info from rpython.jit.metainterp.optimize import InvalidLoop from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.debug import debug_print from rpython.rtyper import rclass from rpython.rtyper.lltypesystem import llmemory from rpython.jit.metainterp.optimize import SpeculativeError @@ -57,9 +60,6 @@ def __init__(self): pass # make rpython happy - def send_extra_operation(self, op, opt=None): - self.optimizer.send_extra_operation(op, opt) - def propagate_forward(self, op): raise NotImplementedError @@ -79,11 +79,11 @@ def emit_extra(self, op, emit=True): if emit: self.emit(op) - self.send_extra_operation(op, self.next_optimization) + self.optimizer.send_extra_operation(op, self.next_optimization) def getintbound(self, op): assert op.type == 'i' - op = self.get_box_replacement(op) + op = get_box_replacement(op) if isinstance(op, ConstInt): return ConstIntBound(op.getint()) fw = op.get_forwarded() @@ -99,7 +99,7 @@ def setintbound(self, op, bound): assert op.type == 'i' - op = self.get_box_replacement(op) + op = get_box_replacement(op) if op.is_constant(): return cur = op.get_forwarded() @@ -111,7 +111,7 @@ def getnullness(self, op): if op.type == 'r' or self.is_raw_ptr(op): - ptrinfo = self.getptrinfo(op) + ptrinfo = getptrinfo(op) if ptrinfo is None: return info.INFO_UNKNOWN return ptrinfo.getnullness() @@ -120,7 +120,7 @@ assert False def make_constant_class(self, op, class_const, update_last_guard=True): - op = self.get_box_replacement(op) + op = op.get_box_replacement() opinfo = op.get_forwarded() if isinstance(opinfo, info.InstancePtrInfo): opinfo._known_class = class_const @@ -136,55 +136,12 @@ opinfo.mark_last_guard(self.optimizer) return opinfo - def getptrinfo(self, op, is_object=False): - if op.type == 'i': - return self.getrawptrinfo(op) - elif op.type == 'f': - return None - assert op.type == 'r' - op = self.get_box_replacement(op) - assert op.type == 'r' - if isinstance(op, ConstPtr): - return info.ConstPtrInfo(op) - fw = op.get_forwarded() - if fw is not None: - assert isinstance(fw, info.PtrInfo) - return fw - return None - def is_raw_ptr(self, op): - fw = self.get_box_replacement(op).get_forwarded() + fw = get_box_replacement(op).get_forwarded() if isinstance(fw, info.AbstractRawPtrInfo): return True return False - def getrawptrinfo(self, op, create=False, is_object=False): - assert op.type == 'i' - op = self.get_box_replacement(op) - assert op.type == 'i' - if isinstance(op, ConstInt): - return info.ConstPtrInfo(op) - fw = op.get_forwarded() - if isinstance(fw, IntBound) and not create: - return None - if fw is not None: - if isinstance(fw, info.AbstractRawPtrInfo): - return fw - fw = info.RawStructPtrInfo() - op.set_forwarded(fw) - assert isinstance(fw, info.AbstractRawPtrInfo) - return fw - return None - - def get_box_replacement(self, op): - return self.optimizer.get_box_replacement(op) - - def getlastop(self): - return self.optimizer.getlastop() - - def force_box(self, op, optforce=None): - return self.optimizer.force_box(op, optforce) - def replace_op_with(self, op, newopnum, args=None, descr=None): return self.optimizer.replace_op_with(op, newopnum, args, descr) @@ -209,18 +166,6 @@ def get_constant_box(self, box): return self.optimizer.get_constant_box(box) - def new_box(self, fieldofs): - return self.optimizer.new_box(fieldofs) - - def new_const(self, fieldofs): - return self.optimizer.new_const(fieldofs) - - def new_box_item(self, arraydescr): - return self.optimizer.new_box_item(arraydescr) - - def new_const_item(self, arraydescr): - return self.optimizer.new_const_item(arraydescr) - def pure(self, opnum, result): if self.optimizer.optpure: self.optimizer.optpure.pure(opnum, result) @@ -229,11 +174,6 @@ if self.optimizer.optpure: self.optimizer.optpure.pure_from_args(opnum, args, op, descr) - def has_pure_result(self, opnum, args, descr): - if self.optimizer.optpure: - return self.optimizer.optpure.has_pure_result(opnum, args, descr) - return False - def get_pure_result(self, key): if self.optimizer.optpure: return self.optimizer.optpure.get_pure_result(key) @@ -308,6 +248,14 @@ self.optimizations = optimizations + def optimize_loop(self, trace, resumestorage, call_pure_results): + traceiter = trace.get_iter() + if resumestorage: + frontend_inputargs = trace.inputargs + deserialize_optimizer_knowledge( + self, resumestorage, frontend_inputargs, traceiter.inputargs) + return self.propagate_all_forward(traceiter, call_pure_results) + def force_op_from_preamble(self, op): return op @@ -322,14 +270,14 @@ def force_box_for_end_of_preamble(self, box): if box.type == 'r': - info = self.getptrinfo(box) + info = getptrinfo(box) if info is not None and info.is_virtual(): rec = {} return info.force_at_the_end_of_preamble(box, self.optearlyforce, rec) return box if box.type == 'i': - info = self.getrawptrinfo(box) + info = getrawptrinfo(box) if info is not None: return info.force_at_the_end_of_preamble(box, self.optearlyforce, None) @@ -345,14 +293,14 @@ def getinfo(self, op): if op.type == 'r': - return self.getptrinfo(op) + return getptrinfo(op) elif op.type == 'i': if self.is_raw_ptr(op): - return self.getptrinfo(op) + return getptrinfo(op) return self.getintbound(op) elif op.type == 'f': - if self.get_box_replacement(op).is_constant(): - return info.FloatConstInfo(self.get_box_replacement(op)) + if get_box_replacement(op).is_constant(): + return info.FloatConstInfo(get_box_replacement(op)) def get_box_replacement(self, op): if op is None: @@ -360,8 +308,9 @@ return op.get_box_replacement() def force_box(self, op, optforce=None): - op = self.get_box_replacement(op) + op = get_box_replacement(op) if optforce is None: + #import pdb; pdb.set_trace() optforce = self info = op.get_forwarded() if self.optunroll and self.optunroll.potential_extra_ops: @@ -387,7 +336,7 @@ return None def get_constant_box(self, box): - box = self.get_box_replacement(box) + box = get_box_replacement(box) if isinstance(box, Const): return box if (box.type == 'i' and box.get_forwarded() and @@ -396,16 +345,8 @@ return None #self.ensure_imported(value) - def get_newoperations(self): - self.flush() - return self._newoperations - - def clear_newoperations(self): - self._newoperations = [] - self._emittedoperations = {} - def make_equal_to(self, op, newop): - op = self.get_box_replacement(op) + op = get_box_replacement(op) if op is newop: return opinfo = op.get_forwarded() @@ -420,7 +361,7 @@ def replace_op_with(self, op, newopnum, args=None, descr=None): newop = op.copy_and_change(newopnum, args, descr) if newop.type != 'v': - op = self.get_box_replacement(op) + op = get_box_replacement(op) opinfo = op.get_forwarded() if opinfo is not None: newop.set_forwarded(opinfo) @@ -429,7 +370,7 @@ def make_constant(self, box, constbox): assert isinstance(constbox, Const) - box = self.get_box_replacement(box) + box = get_box_replacement(box) # safety-check: if the constant is outside the bounds for the # box, then it is an invalid loop if (box.get_forwarded() is not None and @@ -442,7 +383,7 @@ return if box.type == 'r' and box.get_forwarded() is not None: opinfo = box.get_forwarded() - opinfo.copy_fields_to_const(self.getptrinfo(constbox), self.optheap) + opinfo.copy_fields_to_const(getptrinfo(constbox), self.optheap) box.set_forwarded(constbox) def make_constant_int(self, box, intvalue): @@ -662,32 +603,6 @@ self._last_guard_op = None return op - def potentially_change_ovf_op_to_no_ovf(self, op): - # if last emitted operations was int_xxx_ovf and we are not emitting - # a guard_no_overflow change to int_add - if op.getopnum() != rop.GUARD_NO_OVERFLOW: - return - if not self._newoperations: - # got optimized otherwise - return - op = self._newoperations[-1] - if not op.is_ovf(): - return - newop = self.replace_op_with_no_ovf(op) - self._newoperations[-1] = newop - self._emittedoperations[newop] = None - - def replace_op_with_no_ovf(self, op): - if op.getopnum() == rop.INT_MUL_OVF: - return self.replace_op_with(op, rop.INT_MUL) - elif op.getopnum() == rop.INT_ADD_OVF: - return self.replace_op_with(op, rop.INT_ADD) - elif op.getopnum() == rop.INT_SUB_OVF: - return self.replace_op_with(op, rop.INT_SUB) - else: - assert False - - def _copy_resume_data_from(self, guard_op, last_guard_op): last_descr = last_guard_op.getdescr() descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, last_descr) @@ -851,13 +766,8 @@ <= expected_class.subclassrange_max) def is_virtual(self, op): - if op.type == 'r': - opinfo = self.getptrinfo(op) - return opinfo is not None and opinfo.is_virtual() - if op.type == 'i': - opinfo = self.getrawptrinfo(op) - return opinfo is not None and opinfo.is_virtual() - return False + opinfo = getptrinfo(op) + return opinfo is not None and opinfo.is_virtual() # These are typically removed already by OptRewrite, but it can be # dissabled and unrolling emits some SAME_AS ops to setup the @@ -869,4 +779,3 @@ dispatch_opt = make_dispatcher_method(Optimizer, 'optimize_', default=Optimizer.optimize_default) - diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -2,7 +2,8 @@ Optimization, OptimizationResult, REMOVED) from rpython.jit.metainterp.resoperation import rop, OpHelpers, AbstractResOp,\ ResOperation -from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method +from rpython.jit.metainterp.optimizeopt.util import ( + make_dispatcher_method, get_box_replacement) from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp from rpython.jit.metainterp.optimize import SpeculativeError @@ -57,9 +58,9 @@ op = self.lst[i] if op is None: break - if box0.same_box(opt.get_box_replacement(op.getarg(0))) and op.getdescr() is descr: + if box0.same_box(get_box_replacement(op.getarg(0))) and op.getdescr() is descr: op = self.force_preamble_op(opt, op, i) - return opt.get_box_replacement(op) + return get_box_replacement(op) return None def lookup2(self, opt, box0, box1, descr): @@ -67,23 +68,23 @@ op = self.lst[i] if op is None: break - if (box0.same_box(opt.get_box_replacement(op.getarg(0))) and - box1.same_box(opt.get_box_replacement(op.getarg(1))) and + if (box0.same_box(get_box_replacement(op.getarg(0))) and + box1.same_box(get_box_replacement(op.getarg(1))) and op.getdescr() is descr): op = self.force_preamble_op(opt, op, i) - return opt.get_box_replacement(op) + return get_box_replacement(op) return None def lookup(self, optimizer, op): numargs = op.numargs() if numargs == 1: return self.lookup1(optimizer, - optimizer.get_box_replacement(op.getarg(0)), + get_box_replacement(op.getarg(0)), op.getdescr()) elif numargs == 2: return self.lookup2(optimizer, - optimizer.get_box_replacement(op.getarg(0)), - optimizer.get_box_replacement(op.getarg(1)), + get_box_replacement(op.getarg(0)), + get_box_replacement(op.getarg(1)), op.getdescr()) else: assert False @@ -205,7 +206,7 @@ old_start_index = OpHelpers.is_cond_call_value(old_op.opnum) for i in range(old_start_index, old_op.numargs()): box = old_op.getarg(i) - if not self.get_box_replacement(op.getarg(j)).same_box(box): + if not get_box_replacement(op.getarg(j)).same_box(box): break j += 1 else: @@ -239,15 +240,11 @@ def pure_from_args(self, opnum, args, op, descr=None): newop = ResOperation(opnum, - [self.get_box_replacement(arg) for arg in args], + [get_box_replacement(arg) for arg in args], descr=descr) newop.set_forwarded(op) self.pure(opnum, newop) - def has_pure_result(self, opnum, args, descr): - return False - # XXX - def get_pure_result(self, op): recentops = self.getrecentops(op.getopnum()) return recentops.lookup(self.optimizer, op) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -7,13 +7,13 @@ from rpython.jit.metainterp.optimizeopt.intutils import IntBound from rpython.jit.metainterp.optimizeopt.optimizer import ( Optimization, OptimizationResult, REMOVED, CONST_0, CONST_1) -from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL -from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\ - OpHelpers +from rpython.jit.metainterp.optimizeopt.info import ( + INFO_NONNULL, INFO_NULL, getptrinfo) +from rpython.jit.metainterp.optimizeopt.util import ( + _findall, make_dispatcher_method, get_box_replacement) +from rpython.jit.metainterp.resoperation import ( + rop, ResOperation, opclasses, OpHelpers) from rpython.rlib.rarithmetic import highest_bit -from rpython.rtyper.lltypesystem import llmemory -from rpython.rtyper import rclass import math @@ -68,7 +68,6 @@ return True return False - def find_rewritable_bool(self, op): oldopnum = op.boolinverse arg0 = op.getarg(0) @@ -78,7 +77,7 @@ if self.try_boolinvers(op, top): return True - oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL + oldopnum = op.boolreflex # FIXME: add INT_ADD, INT_MUL if oldopnum != -1: top = ResOperation(oldopnum, [arg1, arg0]) oldop = self.get_pure_result(top) @@ -128,8 +127,8 @@ return self.emit(op) def optimize_INT_SUB(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b2.equal(0): @@ -162,9 +161,9 @@ def optimize_INT_ADD(self, op): if self.is_raw_ptr(op.getarg(0)) or self.is_raw_ptr(op.getarg(1)): return self.emit(op) - arg1 = self.get_box_replacement(op.getarg(0)) + arg1 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(arg1) - arg2 = self.get_box_replacement(op.getarg(1)) + arg2 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(arg2) # If one side of the op is 0 the result is the other side. @@ -204,9 +203,9 @@ self.optimizer.pure_from_args(rop.INT_ADD, [inv_arg0, op], arg1) def optimize_INT_MUL(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) + arg1 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(arg1) - arg2 = self.get_box_replacement(op.getarg(1)) + arg2 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(arg2) # If one side of the op is 1 the result is the other side. @@ -276,8 +275,8 @@ # Constant fold f0 * 1.0 and turn f0 * -1.0 into a FLOAT_NEG, these # work in all cases, including NaN and inf for lhs, rhs in [(arg1, arg2), (arg2, arg1)]: - v1 = self.get_box_replacement(lhs) - v2 = self.get_box_replacement(rhs) + v1 = get_box_replacement(lhs) + v2 = get_box_replacement(rhs) if v1.is_constant(): if v1.getfloat() == 1.0: @@ -295,7 +294,7 @@ def optimize_FLOAT_TRUEDIV(self, op): arg1 = op.getarg(0) arg2 = op.getarg(1) - v2 = self.get_box_replacement(arg2) + v2 = get_box_replacement(arg2) # replace "x / const" by "x * (1/const)" if possible newop = op @@ -331,7 +330,7 @@ 'was proven to always fail' % r) return elif box.type == 'r': - box = self.get_box_replacement(box) + box = get_box_replacement(box) if box.is_constant(): if not box.same_constant(constbox): r = self.optimizer.metainterp_sd.logger_ops.repr_of_resop( @@ -343,7 +342,7 @@ return self.emit(op) def optimize_GUARD_ISNULL(self, op): - info = self.getptrinfo(op.getarg(0)) + info = getptrinfo(op.getarg(0)) if info is not None: if info.is_null(): return @@ -357,11 +356,11 @@ self.make_constant(op.getarg(0), CONST_NULL) def optimize_GUARD_IS_OBJECT(self, op): - info = self.getptrinfo(op.getarg(0)) + info = getptrinfo(op.getarg(0)) if info and info.is_constant(): if info.is_null(): raise InvalidLoop("A GUARD_IS_OBJECT(NULL) found") - c = self.get_box_replacement(op.getarg(0)) + c = get_box_replacement(op.getarg(0)) if self.optimizer.cpu.check_is_object(c.getref_base()): return raise InvalidLoop("A GUARD_IS_OBJECT(not-an-object) found") @@ -373,9 +372,9 @@ return self.emit(op) def optimize_GUARD_GC_TYPE(self, op): - info = self.getptrinfo(op.getarg(0)) + info = getptrinfo(op.getarg(0)) if info and info.is_constant(): - c = self.get_box_replacement(op.getarg(0)) + c = get_box_replacement(op.getarg(0)) tid = self.optimizer.cpu.get_actual_typeid(c.getref_base()) if tid != op.getarg(1).getint(): raise InvalidLoop("wrong GC type ID found on a constant") @@ -387,12 +386,12 @@ return self.emit(op) def optimize_GUARD_SUBCLASS(self, op): - info = self.getptrinfo(op.getarg(0)) + info = getptrinfo(op.getarg(0)) optimizer = self.optimizer # must raise 'InvalidLoop' in all cases where 'info' shows the # class cannot possibly match (see test_issue2926) if info and info.is_constant(): - c = self.get_box_replacement(op.getarg(0)) + c = get_box_replacement(op.getarg(0)) vtable = optimizer.cpu.cls_of_box(c).getint() if optimizer._check_subclass(vtable, op.getarg(1).getint()): return @@ -426,7 +425,7 @@ return self.emit(op) From pypy.commits at gmail.com Wed Jun 5 00:38:54 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 04 Jun 2019 21:38:54 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: in python3 crypt -> _crypt and there is a lib-python/3/crypt.py Message-ID: <5cf7475e.1c69fb81.a8f11.95c6@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96741:416b312eb49d Date: 2019-06-05 07:32 +0300 http://bitbucket.org/pypy/pypy/changeset/416b312eb49d/ Log: in python3 crypt -> _crypt and there is a lib-python/3/crypt.py diff --git a/lib_pypy/crypt/__init__.py b/lib_pypy/_crypt/__init__.py rename from lib_pypy/crypt/__init__.py rename to lib_pypy/_crypt/__init__.py --- a/lib_pypy/crypt/__init__.py +++ b/lib_pypy/_crypt/__init__.py @@ -1,5 +1,5 @@ """ -CFFI based implementation of the crypt module +CFFI based implementation of the _crypt module """ import sys From pypy.commits at gmail.com Wed Jun 5 00:38:56 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 04 Jun 2019 21:38:56 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: test for str Message-ID: <5cf74760.1c69fb81.3271f.42e1@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96742:36dc395a0a4a Date: 2019-06-05 07:38 +0300 http://bitbucket.org/pypy/pypy/changeset/36dc395a0a4a/ Log: test for str diff --git a/lib_pypy/_crypt/__init__.py b/lib_pypy/_crypt/__init__.py --- a/lib_pypy/_crypt/__init__.py +++ b/lib_pypy/_crypt/__init__.py @@ -15,6 +15,10 @@ def crypt(word, salt): + if isinstance(word, str): + word = word.encode('ascii') + if isinstance(salt, str): + salt = salt.encode('ascii') res = lib.crypt(word, salt) if not res: return None From pypy.commits at gmail.com Wed Jun 5 01:51:20 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 04 Jun 2019 22:51:20 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge fix-leak-of-file-descriptor-with-_iofile which fixes a resource leak Message-ID: <5cf75858.1c69fb81.3f9ed.0216@mx.google.com> Author: mattip Branch: py3.6 Changeset: r96745:7c2732ad825a Date: 2019-06-05 05:50 +0000 http://bitbucket.org/pypy/pypy/changeset/7c2732ad825a/ Log: merge fix-leak-of-file-descriptor-with-_iofile which fixes a resource leak diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -256,6 +256,7 @@ except: if not fd_is_own: self.fd = -1 + self._close(space) raise def _mode(self): diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -57,8 +57,11 @@ def test_open_directory(self): import _io - import os - raises(IOError, _io.FileIO, self.tmpdir, "rb") + import os, warnings + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', ResourceWarning) + raises(IOError, _io.FileIO, self.tmpdir, "rb") + assert len(w) == 0 if os.name != 'nt': fd = os.open(self.tmpdir, os.O_RDONLY) raises(IOError, _io.FileIO, fd, "rb") From pypy.commits at gmail.com Wed Jun 5 01:51:28 2019 From: pypy.commits at gmail.com (asottile) Date: Tue, 04 Jun 2019 22:51:28 -0700 (PDT) Subject: [pypy-commit] pypy Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863: Fix leak of file descriptor with `_io.FileIO('dir/')` Message-ID: <5cf75860.1c69fb81.55437.635e@mx.google.com> Author: Anthony Sottile Branch: Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863 Changeset: r96743:bb17c9943258 Date: 2019-06-04 22:35 +0000 http://bitbucket.org/pypy/pypy/changeset/bb17c9943258/ Log: Fix leak of file descriptor with `_io.FileIO('dir/')` diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -256,6 +256,7 @@ except: if not fd_is_own: self.fd = -1 + self._close(space) raise def _mode(self): From pypy.commits at gmail.com Wed Jun 5 01:51:30 2019 From: pypy.commits at gmail.com (asottile) Date: Tue, 04 Jun 2019 22:51:30 -0700 (PDT) Subject: [pypy-commit] pypy Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863: add regression test for `FileIO` directory fd leak Message-ID: <5cf75862.1c69fb81.9c870.a41f@mx.google.com> Author: Anthony Sottile Branch: Anthony-Sottile/fix-leak-of-file-descriptor-with-_iofile-1559687440863 Changeset: r96744:d8bc809612bd Date: 2019-06-05 04:35 +0000 http://bitbucket.org/pypy/pypy/changeset/d8bc809612bd/ Log: add regression test for `FileIO` directory fd leak diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -57,8 +57,11 @@ def test_open_directory(self): import _io - import os - raises(IOError, _io.FileIO, self.tmpdir, "rb") + import os, warnings + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', ResourceWarning) + raises(IOError, _io.FileIO, self.tmpdir, "rb") + assert len(w) == 0 if os.name != 'nt': fd = os.open(self.tmpdir, os.O_RDONLY) raises(IOError, _io.FileIO, fd, "rb") From pypy.commits at gmail.com Wed Jun 5 01:56:33 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 04 Jun 2019 22:56:33 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <5cf75991.1c69fb81.32867.4201@mx.google.com> Author: Matti Picus Branch: Changeset: r96746:8ae131ad519d Date: 2019-06-05 08:55 +0300 http://bitbucket.org/pypy/pypy/changeset/8ae131ad519d/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,3 +34,7 @@ .. branch: fix-vmprof-memory-tracking Fix a bug that prevent memory-tracking in vmprof working on PyPy. + +.. branch: optimizeopt-cleanup + +Cleanup optimizeopt From pypy.commits at gmail.com Wed Jun 5 05:32:05 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 02:32:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge default Message-ID: <5cf78c15.1c69fb81.6d21a.faaa@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.6 Changeset: r96747:0ffff10dd2bc Date: 2019-06-05 11:31 +0200 http://bitbucket.org/pypy/pypy/changeset/0ffff10dd2bc/ Log: merge default diff too long, truncating to 2000 out of 7324 lines diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py --- a/extra_tests/cffi_tests/cffi0/test_parsing.py +++ b/extra_tests/cffi_tests/cffi0/test_parsing.py @@ -410,7 +410,17 @@ def test_enum(): ffi = FFI() ffi.cdef(""" - enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1}; + enum Enum { + POS = +1, + TWO = 2, + NIL = 0, + NEG = -1, + ADDSUB = (POS+TWO)-1, + DIVMULINT = (3 * 3) / 2, + SHIFT = (1 << 3) >> 1, + BINOPS = (0x7 & 0x1) | 0x8, + XOR = 0xf ^ 0xa + }; """) needs_dlopen_none() C = ffi.dlopen(None) @@ -418,7 +428,11 @@ assert C.TWO == 2 assert C.NIL == 0 assert C.NEG == -1 - assert C.OP == 2 + assert C.ADDSUB == 2 + assert C.DIVMULINT == 4 + assert C.SHIFT == 4 + assert C.BINOPS == 0b1001 + assert C.XOR == 0b0101 def test_stdcall(): ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py --- a/extra_tests/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -2535,3 +2535,29 @@ x.p = p x.cyclic = x del p, x + +def test_arithmetic_in_cdef(): + for a in [0, 11, 15]: + ffi = FFI() + ffi.cdef(""" + enum FOO { + DIVNN = ((-?) / (-3)), + DIVNP = ((-?) / (+3)), + DIVPN = ((+?) / (-3)), + MODNN = ((-?) % (-3)), + MODNP = ((-?) % (+3)), + MODPN = ((+?) % (-3)), + }; + """.replace('?', str(a))) + lib = ffi.verify(""" + enum FOO { + DIVNN = ((-?) / (-3)), + DIVNP = ((-?) / (+3)), + DIVPN = ((+?) / (-3)), + MODNN = ((-?) % (-3)), + MODNP = ((-?) % (+3)), + MODPN = ((+?) % (-3)), + }; + """.replace('?', str(a))) + # the verify() crashes if the values in the enum are different from + # the values we computed ourselves from the cdef() diff --git a/extra_tests/cffi_tests/cffi0/test_zintegration.py b/extra_tests/cffi_tests/cffi0/test_zintegration.py --- a/extra_tests/cffi_tests/cffi0/test_zintegration.py +++ b/extra_tests/cffi_tests/cffi0/test_zintegration.py @@ -2,11 +2,13 @@ import py, os, sys, shutil import subprocess from extra_tests.cffi_tests.udir import udir +import pytest if sys.platform == 'win32': - py.test.skip('snippets do not run on win32') + pytestmark = pytest.mark.skip('snippets do not run on win32') if sys.version_info < (2, 7): - py.test.skip('fails e.g. on a Debian/Ubuntu which patches virtualenv' + pytestmark = pytest.mark.skip( + 'fails e.g. on a Debian/Ubuntu which patches virtualenv' ' in a non-2.6-friendly way') def create_venv(name): diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -2339,3 +2339,24 @@ typedef int foo_t; struct foo_s { void (*x)(foo_t); }; """) py.test.raises(TypeError, ffi.new, "struct foo_s *") + +def test_from_buffer_struct(): + ffi = FFI() + ffi.cdef("""struct foo_s { int a, b; };""") + lib = verify(ffi, "test_from_buffer_struct_p", """ + struct foo_s { int a, b; }; + """) + p = ffi.new("struct foo_s *", [-219239, 58974983]) + q = ffi.from_buffer("struct foo_s[]", ffi.buffer(p)) + assert ffi.typeof(q) == ffi.typeof("struct foo_s[]") + assert len(q) == 1 + assert q[0].a == p.a + assert q[0].b == p.b + assert q == p + q = ffi.from_buffer("struct foo_s *", ffi.buffer(p)) + assert ffi.typeof(q) == ffi.typeof("struct foo_s *") + assert q.a == p.a + assert q.b == p.b + assert q[0].a == p.a + assert q[0].b == p.b + assert q == p diff --git a/extra_tests/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py --- a/extra_tests/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -64,8 +64,8 @@ output = popen.stdout.read() err = popen.wait() if err: - raise OSError("popen failed with exit code %r: %r" % ( - err, args)) + raise OSError(("popen failed with exit code %r: %r\n\n%s" % ( + err, args, output)).rstrip()) print(output.rstrip()) return output diff --git a/extra_tests/cffi_tests/embedding/test_performance.py b/extra_tests/cffi_tests/embedding/test_performance.py --- a/extra_tests/cffi_tests/embedding/test_performance.py +++ b/extra_tests/cffi_tests/embedding/test_performance.py @@ -3,8 +3,8 @@ from extra_tests.cffi_tests.embedding.test_basic import EmbeddingTests if sys.platform == 'win32': - import py - py.test.skip("written with POSIX functions") + import pytest + pytestmark = pytest.mark.skip("written with POSIX functions") class TestPerformance(EmbeddingTests): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.12.3 +Version: 1.13.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -5,8 +5,8 @@ from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.12.3" -__version_info__ = (1, 12, 3) +__version__ = "1.13.0" +__version_info__ = (1, 13, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -145,6 +145,7 @@ int result; PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; + PyObject *builtins; state = PyGILState_Ensure(); @@ -169,7 +170,7 @@ global_dict = PyDict_New(); if (global_dict == NULL) goto error; - PyObject *builtins = PyEval_GetBuiltins(); + builtins = PyEval_GetBuiltins(); if (builtins == NULL) goto error; if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) @@ -223,7 +224,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.12.3" + "\ncompiled with cffi version: 1.13.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -858,19 +858,39 @@ "the actual array length in this context" % exprnode.coord.line) # - if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and - exprnode.op == '+'): - return (self._parse_constant(exprnode.left) + - self._parse_constant(exprnode.right)) - # - if (isinstance(exprnode, pycparser.c_ast.BinaryOp) and - exprnode.op == '-'): - return (self._parse_constant(exprnode.left) - - self._parse_constant(exprnode.right)) + if isinstance(exprnode, pycparser.c_ast.BinaryOp): + left = self._parse_constant(exprnode.left) + right = self._parse_constant(exprnode.right) + if exprnode.op == '+': + return left + right + elif exprnode.op == '-': + return left - right + elif exprnode.op == '*': + return left * right + elif exprnode.op == '/': + return self._c_div(left, right) + elif exprnode.op == '%': + return left - self._c_div(left, right) * right + elif exprnode.op == '<<': + return left << right + elif exprnode.op == '>>': + return left >> right + elif exprnode.op == '&': + return left & right + elif exprnode.op == '|': + return left | right + elif exprnode.op == '^': + return left ^ right # raise FFIError(":%d: unsupported expression: expected a " "simple numeric constant" % exprnode.coord.line) + def _c_div(self, a, b): + result = a // b + if ((a < 0) ^ (b < 0)) and (a % b) != 0: + result += 1 + return result + def _build_enum_type(self, explicit_name, decls): if decls is not None: partial = False diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -38,3 +38,7 @@ .. branch: fix-vmprof-memory-tracking Fix a bug that prevent memory-tracking in vmprof working on PyPy. + +.. branch: optimizeopt-cleanup + +Cleanup optimizeopt diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.12.3" +VERSION = "1.13.0" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -670,11 +670,14 @@ return self.length def _repr_extra(self): - if self.w_keepalive is not None: - name = self.space.type(self.w_keepalive).name + from pypy.module._cffi_backend import ctypearray + if self.w_keepalive is None: + return "buffer RELEASED" + obj_tp_name = self.space.type(self.w_keepalive).name + if isinstance(self.ctype, ctypearray.W_CTypeArray): + return "buffer len %d from '%s' object" % (self.length, obj_tp_name) else: - name = "(released)" - return "buffer len %d from '%s' object" % (self.length, name) + return "buffer from '%s' object" % (obj_tp_name,) def enter_exit(self, exit_now): # for now, limited effect on PyPy diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -112,9 +112,10 @@ @unwrap_spec(w_ctype=ctypeobj.W_CType, require_writable=int) def from_buffer(space, w_ctype, w_x, require_writable=0): - from pypy.module._cffi_backend import ctypearray - if not isinstance(w_ctype, ctypearray.W_CTypeArray): - raise oefmt(space.w_TypeError, "expected an array ctype, got '%s'", + from pypy.module._cffi_backend import ctypeptr, ctypearray + if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray): + raise oefmt(space.w_TypeError, + "expected a poiunter or array ctype, got '%s'", w_ctype.name) if space.isinstance_w(w_x, space.w_unicode): raise oefmt(space.w_TypeError, @@ -135,33 +136,36 @@ "raw address on PyPy", w_x) # buffersize = buf.getlength() - arraylength = w_ctype.length - if arraylength >= 0: - # it's an array with a fixed length; make sure that the - # buffer contains enough bytes. - if buffersize < w_ctype.size: - raise oefmt(space.w_ValueError, - "buffer is too small (%d bytes) for '%s' (%d bytes)", - buffersize, w_ctype.name, w_ctype.size) + if not isinstance(w_ctype, ctypearray.W_CTypeArray): + arraylength = buffersize # number of bytes, not used so far else: - # it's an open 'array[]' - itemsize = w_ctype.ctitem.size - if itemsize == 1: - # fast path, performance only - arraylength = buffersize - elif itemsize > 0: - # give it as many items as fit the buffer. Ignore a - # partial last element. - arraylength = buffersize / itemsize + arraylength = w_ctype.length + if arraylength >= 0: + # it's an array with a fixed length; make sure that the + # buffer contains enough bytes. + if buffersize < w_ctype.size: + raise oefmt(space.w_ValueError, + "buffer is too small (%d bytes) for '%s' (%d bytes)", + buffersize, w_ctype.name, w_ctype.size) else: - # it's an array 'empty[]'. Unsupported obscure case: - # the problem is that setting the length of the result - # to anything large (like SSIZE_T_MAX) is dangerous, - # because if someone tries to loop over it, it will - # turn effectively into an infinite loop. - raise oefmt(space.w_ZeroDivisionError, - "from_buffer('%s', ..): the actual length of the array " - "cannot be computed", w_ctype.name) + # it's an open 'array[]' + itemsize = w_ctype.ctitem.size + if itemsize == 1: + # fast path, performance only + arraylength = buffersize + elif itemsize > 0: + # give it as many items as fit the buffer. Ignore a + # partial last element. + arraylength = buffersize / itemsize + else: + # it's an array 'empty[]'. Unsupported obscure case: + # the problem is that setting the length of the result + # to anything large (like SSIZE_T_MAX) is dangerous, + # because if someone tries to loop over it, it will + # turn effectively into an infinite loop. + raise oefmt(space.w_ZeroDivisionError, + "from_buffer('%s', ..): the actual length of the array " + "cannot be computed", w_ctype.name) # return cdataobj.W_CDataFromBuffer(space, _cdata, arraylength, w_ctype, buf, w_x) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.12.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.13.0", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -3830,7 +3830,9 @@ BIntP = new_pointer_type(BInt) BIntA = new_array_type(BIntP, None) lst = [-12345678, 87654321, 489148] - bytestring = buffer(newp(BIntA, lst))[:] + b'XYZ' + bytestring = bytearray(buffer(newp(BIntA, lst))[:] + b'XYZ') + lst2 = lst + [42, -999999999] + bytestring2 = bytearray(buffer(newp(BIntA, lst2))[:] + b'XYZ') # p1 = from_buffer(BIntA, bytestring) # int[] assert typeof(p1) is BIntA @@ -3844,7 +3846,19 @@ p1[-1] # py.test.raises(TypeError, from_buffer, BInt, bytestring) - py.test.raises(TypeError, from_buffer, BIntP, bytestring) + # + p2 = from_buffer(BIntP, bytestring) # int * + assert p2 == p1 or 'PY_DOT_PY' in globals() + # note: on py.py ^^^, bytearray buffers are not emulated well enough + assert typeof(p2) is BIntP + assert p2[0] == lst[0] + assert p2[1] == lst[1] + assert p2[2] == lst[2] + # hopefully does not crash, but doesn't raise an exception: + p2[3] + p2[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BIntP, b"") # BIntA2 = new_array_type(BIntP, 2) p2 = from_buffer(BIntA2, bytestring) # int[2] @@ -3856,7 +3870,7 @@ p2[2] with pytest.raises(IndexError): p2[-1] - assert p2 == p1 + assert p2 == p1 or 'PY_DOT_PY' in globals() # BIntA4 = new_array_type(BIntP, 4) # int[4]: too big py.test.raises(ValueError, from_buffer, BIntA4, bytestring) @@ -3866,13 +3880,37 @@ ('a2', BInt, -1)]) BStructP = new_pointer_type(BStruct) BStructA = new_array_type(BStructP, None) - p1 = from_buffer(BStructA, bytestring) # struct[] - assert len(p1) == 1 + p1 = from_buffer(BStructA, bytestring2) # struct[] + assert len(p1) == 2 assert typeof(p1) is BStructA - assert p1[0].a1 == lst[0] - assert p1[0].a2 == lst[1] + assert p1[0].a1 == lst2[0] + assert p1[0].a2 == lst2[1] + assert p1[1].a1 == lst2[2] + assert p1[1].a2 == lst2[3] with pytest.raises(IndexError): - p1[1] + p1[2] + with pytest.raises(IndexError): + p1[-1] + assert repr(p1) == "" + # + p2 = from_buffer(BStructP, bytestring2) # 'struct *' + assert p2 == p1 or 'PY_DOT_PY' in globals() + assert typeof(p2) is BStructP + assert p2.a1 == lst2[0] + assert p2.a2 == lst2[1] + assert p2[0].a1 == lst2[0] + assert p2[0].a2 == lst2[1] + assert p2[1].a1 == lst2[2] + assert p2[1].a2 == lst2[3] + # does not crash: + p2[2] + p2[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BStructP, b"") + from_buffer(BStructP, b"1234567") + # + release(p1) + assert repr(p1) == "" # BEmptyStruct = new_struct_type("empty") complete_struct_or_union(BEmptyStruct, [], Ellipsis, 0) @@ -3886,7 +3924,37 @@ p1 = from_buffer(BEmptyStructA5, bytestring) # struct empty[5] assert typeof(p1) is BEmptyStructA5 assert len(p1) == 5 - assert cast(BIntP, p1) == from_buffer(BIntA, bytestring) + assert (cast(BIntP, p1) == from_buffer(BIntA, bytestring) + or 'PY_DOT_PY' in globals()) + # + BVarStruct = new_struct_type("varfoo") + BVarStructP = new_pointer_type(BVarStruct) + complete_struct_or_union(BVarStruct, [('a1', BInt, -1), + ('va', BIntA, -1)]) + with pytest.raises(TypeError): + from_buffer(BVarStruct, bytestring) + pv = from_buffer(BVarStructP, bytestring) # varfoo * + assert pv.a1 == lst[0] + assert pv.va[0] == lst[1] + assert pv.va[1] == lst[2] + assert sizeof(pv[0]) == 1 * size_of_int() + with pytest.raises(TypeError): + len(pv.va) + # hopefully does not crash, but doesn't raise an exception: + pv.va[2] + pv.va[-1] + # not enough data even for one, but this is not enforced: + from_buffer(BVarStructP, b"") + assert repr(pv) == "" + assert repr(pv[0]).startswith("" + assert repr(pv[0]).startswith("" release(p) assert p[2] == b"z" # true so far, but might change to raise RuntimeError + assert repr(p) == "" release(p) # no effect def test_explicit_release_from_buffer_contextmgr(): @@ -4325,6 +4395,7 @@ with p: assert p[2] == b"z" assert p[2] == b"z" # true so far, but might change to raise RuntimeError + assert repr(p) == "" release(p) # no effect def test_explicit_release_bytearray_on_cpython(): diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -2,6 +2,8 @@ import sys +import sys + from rpython.rlib.objectmodel import ( compute_hash, compute_unique_id, import_from_mixin, always_inline, enforceargs, newlist_hint, specialize, we_are_translated) diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst --- a/rpython/doc/jit/optimizer.rst +++ b/rpython/doc/jit/optimizer.rst @@ -13,7 +13,7 @@ Before some optimizations are explained in more detail, it is essential to understand how traces look like. -The optimizer comes with a test suit. It contains many trace +The optimizer comes with a test suite. It contains many trace examples and you might want to take a look at it (in `rpython/jit/metainterp/optimizeopt/test/*.py`). The allowed operations can be found in `rpython/jit/metainterp/resoperation.py`. @@ -21,7 +21,7 @@ [p0,i0,i1] label(p0, i0, i1) - i2 = getarray_item_raw(p0, i0, descr=) + i2 = getarrayitem_raw(p0, i0, descr=) i3 = int_add(i1,i2) i4 = int_add(i0,1) i5 = int_le(i4, 100) # lower-or-equal @@ -32,7 +32,7 @@ to compare the Python code that constructed the trace:: from array import array - a = array('i',range(101)) + a = array('i', range(101)) sum = 0; i = 0 while i <= 100: # can be seen as label sum += a[i] @@ -131,20 +131,16 @@ Whenever such an operation is encountered (e.g. ``y = x & 0``), no operation is emitted. Instead the variable y is made equal to 0 -(= ``make_equal_to(op.result, 0)``). The variables found in a trace are -instances of Box classes that can be found in -`rpython/jit/metainterp/history.py`. `OptValue` wraps those variables again -and maps the boxes to the optimization values in the optimizer. When a -value is made equal, the two variable's boxes are made to point to the same -`OptValue` instance. +(= ``make_constant_int(op, 0)``). The variables found in a trace are instances +of classes that can be found in `rpython/jit/metainterp/history.py`. When a +value is made equal to another, its box is made to point to the other one. -**NOTE: this OptValue organization is currently being refactored in a branch.** Pure optimization ----------------- -Is interwoven into the basic optimizer. It saves operations, results, -arguments to be known to have pure semantics. +The 'pure' optimizations interwoven into the basic optimizer. It saves +operations, results, arguments to be known to have pure semantics. "Pure" here means the same as the ``jit.elidable`` decorator: free of "observable" side effects and referentially transparent diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -3,23 +3,22 @@ from rpython.rtyper.annlowlevel import ( cast_instance_to_gcref, cast_gcref_to_instance) from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.debug import debug_start, debug_stop, debug_print, have_debug_prints +from rpython.rlib.debug import ( + debug_start, debug_stop, debug_print, have_debug_prints) from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib import rstack from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside from rpython.rlib.rjitlog import rjitlog as jl -from rpython.rlib.objectmodel import compute_unique_id -from rpython.conftest import option -from rpython.jit.metainterp.resoperation import ResOperation, rop,\ - get_deep_immutable_oplist, OpHelpers, InputArgInt, InputArgRef,\ - InputArgFloat -from rpython.jit.metainterp.history import (TreeLoop, Const, JitCellToken, +from rpython.jit.metainterp.resoperation import ( + ResOperation, rop, get_deep_immutable_oplist, OpHelpers, InputArgInt, + InputArgRef, InputArgFloat) +from rpython.jit.metainterp.history import (TreeLoop, JitCellToken, TargetToken, AbstractFailDescr, ConstInt) from rpython.jit.metainterp import history, jitexc from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.resume import (PENDINGFIELDSP, - ResumeDataDirectReader, AccumInfo) +from rpython.jit.metainterp.resume import ( + PENDINGFIELDSP, ResumeDataDirectReader) from rpython.jit.metainterp.resumecode import NUMBERING from rpython.jit.metainterp.support import adr2int from rpython.jit.codewriter import longlong @@ -30,6 +29,9 @@ raise SwitchToBlackhole(Counters.ABORT_BRIDGE) class CompileData(object): + """ An object that accumulates all of the necessary info for + the optimization phase, but does not actually have any other state + """ memo = None log_noopt = True @@ -37,10 +39,28 @@ for arg in self.trace.inputargs: arg.set_forwarded(None) -class LoopCompileData(CompileData): - """ An object that accumulates all of the necessary info for - the optimization phase, but does not actually have any other state + def optimize_trace(self, metainterp_sd, jitdriver_sd, memo): + """Optimize loop.operations to remove internal overheadish operations. + """ + from rpython.jit.metainterp.optimizeopt import build_opt_chain + # mark that a new trace has been started + log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE, metainterp_sd, None) + log.write_trace(self.trace) + if self.log_noopt: + metainterp_sd.logger_noopt.log_loop_from_trace(self.trace, memo=memo) + self.box_names_memo = memo + optimizations = build_opt_chain(self.enable_opts) + debug_start("jit-optimize") + try: + return self.optimize(metainterp_sd, jitdriver_sd, optimizations) + finally: + self.forget_optimization_info() + debug_stop("jit-optimize") + + +class PreambleCompileData(CompileData): + """ This is the case of label() ops label() """ def __init__(self, trace, runtime_boxes, call_pure_results=None, @@ -51,19 +71,12 @@ assert runtime_boxes is not None self.runtime_boxes = runtime_boxes - def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): - from rpython.jit.metainterp.optimizeopt.unroll import (UnrollOptimizer, - Optimizer) - - if unroll: - opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.optimize_preamble(self.trace, - self.runtime_boxes, - self.call_pure_results, - self.box_names_memo) - else: - opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.propagate_all_forward(self.trace, self.call_pure_results) + def optimize(self, metainterp_sd, jitdriver_sd, optimizations): + from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer + opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) + return opt.optimize_preamble( + self.trace, self.runtime_boxes, self.call_pure_results, + self.box_names_memo) class SimpleCompileData(CompileData): """ This represents label() ops jump with no extra info associated with @@ -76,26 +89,19 @@ self.call_pure_results = call_pure_results self.enable_opts = enable_opts - def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): + def optimize(self, metainterp_sd, jitdriver_sd, optimizations): from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer - from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge - - #assert not unroll opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations) - traceiter = self.trace.get_iter() - if self.resumestorage: - frontend_inputargs = self.trace.inputargs - deserialize_optimizer_knowledge(opt, self.resumestorage, - frontend_inputargs, - traceiter.inputargs) - return opt.propagate_all_forward(traceiter, self.call_pure_results) + return opt.optimize_loop( + self.trace, self.resumestorage, self.call_pure_results) class BridgeCompileData(CompileData): """ This represents ops() with a jump at the end that goes to some loop, we need to deal with virtual state and inlining of short preamble """ - def __init__(self, trace, runtime_boxes, resumestorage=None, call_pure_results=None, - enable_opts=None, inline_short_preamble=False): + def __init__(self, trace, runtime_boxes, resumestorage=None, + call_pure_results=None, enable_opts=None, + inline_short_preamble=False): self.trace = trace self.runtime_boxes = runtime_boxes self.call_pure_results = call_pure_results @@ -103,7 +109,7 @@ self.inline_short_preamble = inline_short_preamble self.resumestorage = resumestorage - def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): + def optimize(self, metainterp_sd, jitdriver_sd, optimizations): from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) @@ -115,29 +121,26 @@ class UnrolledLoopData(CompileData): """ This represents label() ops jump with extra info that's from the - run of LoopCompileData. Jump goes to the same label + run of PreambleCompileData. Jump goes to the same label """ log_noopt = False def __init__(self, trace, celltoken, state, - call_pure_results=None, enable_opts=None, - inline_short_preamble=True): + call_pure_results=None, enable_opts=None): self.trace = trace self.celltoken = celltoken self.enable_opts = enable_opts self.state = state self.call_pure_results = call_pure_results - self.inline_short_preamble = inline_short_preamble - def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): + def optimize(self, metainterp_sd, jitdriver_sd, optimizations): from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer - - assert unroll # we should not be here if it's disabled opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state, - self.call_pure_results, self.inline_short_preamble) + return opt.optimize_peeled_loop( + self.trace, self.celltoken, self.state, self.call_pure_results) def show_procedures(metainterp_sd, procedure=None, error=None): + from rpython.conftest import option # debugging if option and (option.view or option.viewloops): if error: @@ -173,19 +176,14 @@ original_jitcell_token = loop.original_jitcell_token assert original_jitcell_token is not None if metainterp_sd.warmrunnerdesc is not None: # for tests - assert original_jitcell_token.generation > 0 # has been registered with memmgr + assert original_jitcell_token.generation > 0 # has been registered with memmgr wref = weakref.ref(original_jitcell_token) clt = original_jitcell_token.compiled_loop_token clt.loop_token_wref = wref for op in loop.operations: descr = op.getdescr() - # not sure what descr.index is about if isinstance(descr, ResumeDescr): descr.rd_loop_token = clt # stick it there - #n = descr.index - #if n >= 0: # we also record the resumedescr number - # original_jitcell_token.compiled_loop_token.record_faildescr_index(n) - # pass if isinstance(descr, JitCellToken): # for a CALL_ASSEMBLER: record it as a potential jump. if descr is not original_jitcell_token: @@ -215,11 +213,8 @@ # ____________________________________________________________ - def compile_simple_loop(metainterp, greenkey, trace, runtime_args, enable_opts, cut_at): - from rpython.jit.metainterp.optimizeopt import optimize_trace - jitdriver_sd = metainterp.jitdriver_sd metainterp_sd = metainterp.staticdata jitcell_token = make_jitcell_token(jitdriver_sd) @@ -227,8 +222,8 @@ data = SimpleCompileData(trace, call_pure_results=call_pure_results, enable_opts=enable_opts) try: - loop_info, ops = optimize_trace(metainterp_sd, jitdriver_sd, - data, metainterp.box_names_memo) + loop_info, ops = data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: metainterp_sd.jitlog.trace_aborted() trace.cut_at(cut_at) @@ -253,12 +248,10 @@ return target_token def compile_loop(metainterp, greenkey, start, inputargs, jumpargs, - full_preamble_needed=True, try_disabling_unroll=False): + use_unroll=True): """Try to compile a new procedure by closing the current history back to the first operation. """ - from rpython.jit.metainterp.optimizeopt import optimize_trace - metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd history = metainterp.history @@ -269,28 +262,21 @@ faildescr=None, entry_bridge=False) # enable_opts = jitdriver_sd.warmstate.enable_opts - if try_disabling_unroll: - if 'unroll' not in enable_opts: - return None - enable_opts = enable_opts.copy() - del enable_opts['unroll'] - jitcell_token = make_jitcell_token(jitdriver_sd) cut_at = history.get_trace_position() history.record(rop.JUMP, jumpargs, None, descr=jitcell_token) if start != (0, 0, 0): trace = trace.cut_trace_from(start, inputargs) - if 'unroll' not in enable_opts or not metainterp.cpu.supports_guard_gc_type: + if not use_unroll: return compile_simple_loop(metainterp, greenkey, trace, jumpargs, enable_opts, cut_at) call_pure_results = metainterp.call_pure_results - preamble_data = LoopCompileData(trace, jumpargs, + preamble_data = PreambleCompileData(trace, jumpargs, call_pure_results=call_pure_results, enable_opts=enable_opts) try: - start_state, preamble_ops = optimize_trace(metainterp_sd, jitdriver_sd, - preamble_data, - metainterp.box_names_memo) + start_state, preamble_ops = preamble_data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: metainterp_sd.jitlog.trace_aborted() history.cut(cut_at) @@ -305,9 +291,8 @@ call_pure_results=call_pure_results, enable_opts=enable_opts) try: - loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd, - loop_data, - metainterp.box_names_memo) + loop_info, loop_ops = loop_data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: metainterp_sd.jitlog.trace_aborted() history.cut(cut_at) @@ -354,8 +339,6 @@ """Try to compile a new procedure by closing the current history back to the first operation. """ - from rpython.jit.metainterp.optimizeopt import optimize_trace - trace = metainterp.history.trace.cut_trace_from(start, inputargs) metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd @@ -375,29 +358,14 @@ call_pure_results=call_pure_results, enable_opts=enable_opts) try: - loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd, - loop_data, - metainterp.box_names_memo) + loop_info, loop_ops = loop_data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: - # Fall back on jumping directly to preamble + metainterp_sd.jitlog.trace_aborted() history.cut(cut) - history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token) - loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state, - call_pure_results=call_pure_results, - enable_opts=enable_opts, - inline_short_preamble=False) - try: - loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd, - loop_data, - metainterp.box_names_memo) - except InvalidLoop: - metainterp_sd.jitlog.trace_aborted() - history.cut(cut) - return None + return None label_op = loop_info.label_op - if label_op is None: - assert False, "unreachable code" # hint for some strange tests label_token = label_op.getdescr() assert isinstance(label_token, TargetToken) if label_token.short_preamble: @@ -405,9 +373,8 @@ label_token.short_preamble, metainterp.box_names_memo) loop = partial_trace loop.original_jitcell_token = loop_jitcell_token - loop.operations = (loop.operations + loop_info.extra_same_as + - [loop_info.label_op] - + loop_ops) + loop.operations = ( + loop.operations + loop_info.extra_same_as + [label_op] + loop_ops) quasi_immutable_deps = {} if loop_info.quasi_immutable_deps: @@ -419,8 +386,6 @@ target_token = loop.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, loop, inputargs) - - record_loop_or_bridge(metainterp_sd, loop) return target_token def get_box_replacement(op, allow_none=False): @@ -828,6 +793,7 @@ self, inputargs, new_loop.operations, new_loop.original_jitcell_token, metainterp.box_names_memo) + record_loop_or_bridge(metainterp.staticdata, new_loop) def make_a_counter_per_value(self, guard_value_op, index): assert guard_value_op.getopnum() == rop.GUARD_VALUE @@ -1038,18 +1004,16 @@ jitdriver_sd.warmstate.attach_procedure_to_interp( self.original_greenkey, jitcell_token) metainterp_sd.stats.add_jitcell_token(jitcell_token) + record_loop_or_bridge(metainterp_sd, new_loop) def get_resumestorage(self): return None -def compile_trace(metainterp, resumekey, runtime_boxes): +def compile_trace(metainterp, resumekey, runtime_boxes, ends_with_jump=False): """Try to compile a new bridge leading from the beginning of the history to some existing place. """ - - from rpython.jit.metainterp.optimizeopt import optimize_trace - # The history contains new operations to attach as the code for the # failure of 'resumekey.guard_op'. # @@ -1058,11 +1022,6 @@ metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd - # - jd_name = jitdriver_sd.jitdriver.name - metainterp_sd.jitlog.start_new_trace(metainterp_sd, - faildescr=resumekey, entry_bridge=False, jd_name=jd_name) - # if isinstance(resumekey, ResumeAtPositionDescr): inline_short_preamble = False else: @@ -1071,11 +1030,15 @@ trace = metainterp.history.trace jitdriver_sd = metainterp.jitdriver_sd enable_opts = jitdriver_sd.warmstate.enable_opts - call_pure_results = metainterp.call_pure_results resumestorage = resumekey.get_resumestorage() - if metainterp.history.ends_with_jump: + trace.tracing_done() + metainterp_sd.jitlog.start_new_trace(metainterp_sd, + faildescr=resumekey, entry_bridge=False, + jd_name=jitdriver_sd.jitdriver.name) + + if ends_with_jump: data = BridgeCompileData(trace, runtime_boxes, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts, @@ -1085,12 +1048,10 @@ call_pure_results=call_pure_results, enable_opts=enable_opts) try: - info, newops = optimize_trace(metainterp_sd, jitdriver_sd, - data, metainterp.box_names_memo) + info, newops = data.optimize_trace( + metainterp_sd, jitdriver_sd, metainterp.box_names_memo) except InvalidLoop: metainterp_sd.jitlog.trace_aborted() - #pdb.post_mortem(sys.exc_info()[2]) - debug_print("compile_new_bridge: got an InvalidLoop") # XXX I am fairly convinced that optimize_bridge cannot actually raise # InvalidLoop debug_print('InvalidLoop in compile_new_bridge') @@ -1104,7 +1065,6 @@ new_trace.inputargs = info.inputargs target_token = new_trace.operations[-1].getdescr() resumekey.compile_and_attach(metainterp, new_trace, inputargs) - record_loop_or_bridge(metainterp_sd, new_trace) return target_token new_trace.inputargs = info.renamed_inputargs metainterp.retrace_needed(new_trace, info) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -689,7 +689,6 @@ class History(object): - ends_with_jump = False trace = None def __init__(self): diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py --- a/rpython/jit/metainterp/opencoder.py +++ b/rpython/jit/metainterp/opencoder.py @@ -7,12 +7,13 @@ """ from rpython.jit.metainterp.history import ( - ConstInt, Const, ConstFloat, ConstPtr, new_ref_dict) + ConstInt, Const, ConstFloat, ConstPtr, new_ref_dict, SwitchToBlackhole) from rpython.jit.metainterp.resoperation import AbstractResOp, AbstractInputArg,\ ResOperation, oparity, rop, opwithdescr, GuardResOp, IntOp, FloatOp, RefOp,\ opclasses from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib.objectmodel import we_are_translated, specialize +from rpython.rlib.jit import Counters from rpython.rtyper.lltypesystem import rffi, lltype, llmemory TAGINT, TAGCONSTPTR, TAGCONSTOTHER, TAGBOX = range(4) @@ -301,7 +302,8 @@ def tracing_done(self): from rpython.rlib.debug import debug_start, debug_stop, debug_print - assert not self.tag_overflow + if self.tag_overflow: + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) self._bigints_dict = {} self._refs_dict = new_ref_dict() diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py --- a/rpython/jit/metainterp/optimizeopt/__init__.py +++ b/rpython/jit/metainterp/optimizeopt/__init__.py @@ -1,4 +1,3 @@ -from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer from rpython.jit.metainterp.optimizeopt.rewrite import OptRewrite from rpython.jit.metainterp.optimizeopt.intbounds import OptIntBounds from rpython.jit.metainterp.optimizeopt.virtualize import OptVirtualize @@ -7,10 +6,8 @@ from rpython.jit.metainterp.optimizeopt.simplify import OptSimplify from rpython.jit.metainterp.optimizeopt.pure import OptPure from rpython.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce -from rpython.rlib.rjitlog import rjitlog as jl from rpython.rlib.jit import PARAMETERS, ENABLE_ALL_OPTS from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib.debug import debug_start, debug_stop, debug_print ALL_OPTS = [('intbounds', OptIntBounds), @@ -31,45 +28,17 @@ assert ENABLE_ALL_OPTS == ALL_OPTS_NAMES, ( 'please fix rlib/jit.py to say ENABLE_ALL_OPTS = %r' % (ALL_OPTS_NAMES,)) -def build_opt_chain(metainterp_sd, enable_opts): +def build_opt_chain(enable_opts): optimizations = [] - unroll = 'unroll' in enable_opts # 'enable_opts' is normally a dict - if (metainterp_sd.cpu is not None and - not metainterp_sd.cpu.supports_guard_gc_type): - unroll = False for name, opt in unroll_all_opts: if name in enable_opts: if opt is not None: o = opt() optimizations.append(o) - - if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts - or 'heap' not in enable_opts or 'pure' not in enable_opts): - optimizations.append(OptSimplify(unroll)) - - return optimizations, unroll - -def optimize_trace(metainterp_sd, jitdriver_sd, compile_data, memo=None): - """Optimize loop.operations to remove internal overheadish operations. - """ - debug_start("jit-optimize") - try: - # mark that a new trace has been started - log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE, metainterp_sd, None) - log.write_trace(compile_data.trace) - if compile_data.log_noopt: - metainterp_sd.logger_noopt.log_loop_from_trace(compile_data.trace, memo=memo) - if memo is None: - memo = {} - compile_data.box_names_memo = memo - optimizations, unroll = build_opt_chain(metainterp_sd, - compile_data.enable_opts) - return compile_data.optimize(metainterp_sd, jitdriver_sd, - optimizations, unroll) - finally: - compile_data.forget_optimization_info() - debug_stop("jit-optimize") + if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts or + 'heap' not in enable_opts or 'pure' not in enable_opts): + optimizations.append(OptSimplify()) + return optimizations if __name__ == '__main__': print ALL_OPTS_NAMES - diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -3,6 +3,7 @@ from rpython.jit.metainterp import resumecode from rpython.jit.metainterp.history import Const, ConstInt, CONST_NULL +from .info import getptrinfo # adds the following sections at the end of the resume code: @@ -75,7 +76,7 @@ for box in liveboxes: if box is None or box.type != "r": continue - info = optimizer.getptrinfo(box) + info = getptrinfo(box) known_class = info is not None and info.get_known_class(optimizer.cpu) is not None bitfield <<= 1 bitfield |= known_class diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -1,26 +1,19 @@ -import os from collections import OrderedDict from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.metainterp.optimizeopt.util import args_dict -from rpython.jit.metainterp.history import Const, ConstInt, new_ref_dict -from rpython.jit.metainterp.jitexc import JitException +from rpython.jit.metainterp.history import new_ref_dict from rpython.jit.metainterp.optimizeopt.optimizer import Optimization, REMOVED -from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method +from rpython.jit.metainterp.optimizeopt.util import ( + make_dispatcher_method, get_box_replacement) from rpython.jit.metainterp.optimizeopt.intutils import IntBound from rpython.jit.metainterp.optimizeopt.shortpreamble import PreambleOp from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\ - GuardResOp +from rpython.jit.metainterp.resoperation import rop from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.optimizeopt import info - -class BogusImmutableField(JitException): - pass - - class AbstractCachedEntry(object): """ abstract base class abstracting over the difference between caching struct fields and array items. """ @@ -56,40 +49,32 @@ descr, index=-1): assert self._lazy_set is None for i, info in enumerate(self.cached_infos): - structbox = optimizer.get_box_replacement(self.cached_structs[i]) + structbox = get_box_replacement(self.cached_structs[i]) info.produce_short_preamble_ops(structbox, descr, index, optimizer, shortboxes) - def possible_aliasing(self, optheap, opinfo): + def possible_aliasing(self, opinfo): # If lazy_set is set and contains a setfield on a different # structvalue, then we are annoyed, because it may point to either # the same or a different structure at runtime. # XXX constants? return (self._lazy_set is not None - and (not optheap.getptrinfo( - self._lazy_set.getarg(0)).same_info(opinfo))) + and not info.getptrinfo(self._lazy_set.getarg(0)).same_info(opinfo)) def do_setfield(self, optheap, op): # Update the state with the SETFIELD_GC/SETARRAYITEM_GC operation 'op'. structinfo = optheap.ensure_ptr_info_arg0(op) - arg1 = optheap.get_box_replacement(self._get_rhs_from_set_op(op)) - if self.possible_aliasing(optheap, structinfo): + arg1 = get_box_replacement(self._get_rhs_from_set_op(op)) + if self.possible_aliasing(structinfo): self.force_lazy_set(optheap, op.getdescr()) - assert not self.possible_aliasing(optheap, structinfo) + assert not self.possible_aliasing(structinfo) cached_field = self._getfield(structinfo, op.getdescr(), optheap, False) if cached_field is not None: - cached_field = optheap.get_box_replacement(cached_field) - - # Hack to ensure constants are imported from the preamble - # XXX no longer necessary? - #if cached_fieldvalue and fieldvalue.is_constant(): - # optheap.optimizer.ensure_imported(cached_fieldvalue) - # cached_fieldvalue = self._cached_fields.get(structvalue, None) + cached_field = cached_field.get_box_replacement() if not cached_field or not cached_field.same_box(arg1): # common case: store the 'op' as lazy_set self._lazy_set = op - else: # this is the case where the pending setfield ends up # storing precisely the value that is already there, @@ -104,11 +89,11 @@ def getfield_from_cache(self, optheap, opinfo, descr): # Returns the up-to-date field's value, or None if not cached. - if self.possible_aliasing(optheap, opinfo): + if self.possible_aliasing(opinfo): self.force_lazy_set(optheap, descr) if self._lazy_set is not None: op = self._lazy_set - return optheap.get_box_replacement(self._get_rhs_from_set_op(op)) + return get_box_replacement(self._get_rhs_from_set_op(op)) else: res = self._getfield(opinfo, descr, optheap) if res is not None: @@ -140,7 +125,6 @@ elif not can_cache: self.invalidate(descr) - # abstract methods def _get_rhs_from_set_op(self, op): @@ -167,8 +151,8 @@ return op.getarg(1) def put_field_back_to_info(self, op, opinfo, optheap): - arg = optheap.get_box_replacement(op.getarg(1)) - struct = optheap.get_box_replacement(op.getarg(0)) + arg = get_box_replacement(op.getarg(1)) + struct = get_box_replacement(op.getarg(0)) opinfo.setfield(op.getdescr(), struct, arg, optheap=optheap, cf=self) def _getfield(self, opinfo, descr, optheap, true_force=True): @@ -216,8 +200,8 @@ return res def put_field_back_to_info(self, op, opinfo, optheap): - arg = optheap.get_box_replacement(op.getarg(2)) - struct = optheap.get_box_replacement(op.getarg(0)) + arg = get_box_replacement(op.getarg(2)) + struct = get_box_replacement(op.getarg(0)) opinfo.setitem(op.getdescr(), self.index, struct, arg, optheap=optheap, cf=self) def invalidate(self, descr): @@ -412,8 +396,8 @@ d = self.cached_dict_reads[descr1] = args_dict() self.corresponding_array_descrs[descrs[1]] = descr1 # - key = [self.optimizer.get_box_replacement(op.getarg(1)), # dict - self.optimizer.get_box_replacement(op.getarg(2))] # key + key = [get_box_replacement(op.getarg(1)), # dict + get_box_replacement(op.getarg(2))] # key # other args can be ignored here (hash, store_flag) try: res_v = d[key] @@ -525,7 +509,7 @@ # guards' resume data is that of a virtual object that is stored # into a field of a non-virtual object. Here, 'op' in either # SETFIELD_GC or SETARRAYITEM_GC. - opinfo = self.getptrinfo(op.getarg(0)) + opinfo = info.getptrinfo(op.getarg(0)) assert not opinfo.is_virtual() # it must be a non-virtual if self.optimizer.is_virtual(op.getarg(2)): pendingfields.append(op) @@ -564,13 +548,6 @@ def optimize_SETFIELD_GC(self, op): self.setfield(op) - #opnum = OpHelpers.getfield_pure_for_descr(op.getdescr()) - #if self.has_pure_result(opnum, [op.getarg(0)], - # op.getdescr()): - # os.write(2, '[bogus _immutable_field_ declaration: %s]\n' % - # (op.getdescr().repr_of_descr())) - # raise BogusImmutableField - # def setfield(self, op): cf = self.field_cache(op.getdescr()) @@ -606,8 +583,8 @@ index = indexb.getint() cf = self.arrayitem_cache(op.getdescr(), index) arrayinfo.setitem(op.getdescr(), indexb.getint(), - self.get_box_replacement(op.getarg(0)), - self.get_box_replacement(op), optheap=self, + get_box_replacement(op.getarg(0)), + get_box_replacement(op), optheap=self, cf=cf) optimize_GETARRAYITEM_GC_R = optimize_GETARRAYITEM_GC_I optimize_GETARRAYITEM_GC_F = optimize_GETARRAYITEM_GC_I @@ -639,13 +616,6 @@ optimize_GETARRAYITEM_GC_PURE_F = optimize_GETARRAYITEM_GC_PURE_I def optimize_SETARRAYITEM_GC(self, op): - #opnum = OpHelpers.getarrayitem_pure_for_descr(op.getdescr()) - #if self.has_pure_result(opnum, [op.getarg(0), op.getarg(1)], - # op.getdescr()): - # os.write(2, '[bogus immutable array declaration: %s]\n' % - # (op.getdescr().repr_of_descr())) - # raise BogusImmutableField - # indexb = self.getintbound(op.getarg(1)) if indexb.is_constant(): arrayinfo = self.ensure_ptr_info_arg0(op) @@ -680,8 +650,7 @@ # check that the value is still correct; it could have changed # already between the tracing and now. In this case, we mark the loop # as invalid - if not qmutdescr.is_still_valid_for( - self.get_box_replacement(op.getarg(0))): + if not qmutdescr.is_still_valid_for(get_box_replacement(op.getarg(0))): raise InvalidLoop('quasi immutable field changed during tracing') # record as an out-of-line guard if self.optimizer.quasi_immutable_deps is None: @@ -701,10 +670,10 @@ result_getfield = [] for descr, cf in self.cached_fields.iteritems(): if cf._lazy_set: - continue # XXX safe default for now + continue # XXX safe default for now parent_descr = descr.get_parent_descr() if not parent_descr.is_object(): - continue # XXX could be extended to non-instance objects + continue # XXX could be extended to non-instance objects for i, box1 in enumerate(cf.cached_structs): if not box1.is_constant() and box1 not in available_boxes: continue @@ -721,7 +690,7 @@ for descr, indexdict in self.cached_arrayitems.iteritems(): for index, cf in indexdict.iteritems(): if cf._lazy_set: - continue # XXX safe default for now + continue # XXX safe default for now for i, box1 in enumerate(cf.cached_structs): if not box1.is_constant() and box1 not in available_boxes: continue diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -1,13 +1,14 @@ -from rpython.rlib.objectmodel import specialize, we_are_translated, compute_hash -from rpython.jit.metainterp.resoperation import AbstractValue, ResOperation,\ - rop, OpHelpers -from rpython.jit.metainterp.history import ConstInt, Const -from rpython.rtyper.lltypesystem import lltype, llmemory -from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation +from rpython.rlib.objectmodel import specialize, compute_hash +from rpython.jit.metainterp.resoperation import ( + AbstractValue, ResOperation, rop, OpHelpers) +from rpython.jit.metainterp.history import ConstInt, ConstPtr, Const +from rpython.rtyper.lltypesystem import lltype +from rpython.jit.metainterp.optimizeopt.rawbuffer import ( + RawBuffer, InvalidRawOperation) from rpython.jit.metainterp.executor import execute from rpython.jit.metainterp.optimize import InvalidLoop - +from .util import get_box_replacement INFO_NULL = 0 INFO_NONNULL = 1 @@ -51,7 +52,7 @@ def force_at_the_end_of_preamble(self, op, optforce, rec): if not self.is_virtual(): - return optforce.get_box_replacement(op) + return get_box_replacement(op) return self._force_at_the_end_of_preamble(op, optforce, rec) def get_known_class(self, cpu): @@ -84,7 +85,7 @@ @specialize.arg(2) def get_constant_string_spec(self, string_optimizer, mode): - return None # can't be constant + return None # can't be constant class NonNullPtrInfo(PtrInfo): _attrs_ = ('last_guard_pos',) @@ -109,7 +110,7 @@ def mark_last_guard(self, optimizer): if (optimizer.getlastop() is None or - not optimizer.getlastop().is_guard()): + not optimizer.getlastop().is_guard()): # there can be a really emitted operation that's not a guard # e.g. a setfield, ignore those return @@ -140,12 +141,12 @@ constptr = optforce.optimizer.constant_fold(op) op.set_forwarded(constptr) self._is_virtual = False - self._force_elements_immutable(self.descr, constptr, optforce) + self._force_elements_immutable(self.descr, constptr, optforce.optimizer) return constptr # op.set_forwarded(None) optforce.emit_extra(op) - newop = optforce.getlastop() + newop = optforce.optimizer.getlastop() if newop is not op: op.set_forwarded(newop) newop.set_forwarded(self) @@ -161,14 +162,14 @@ def is_virtual(self): return self._is_virtual - def _visitor_walk_recursive(self, op, visitor, optimizer): + def _visitor_walk_recursive(self, op, visitor): raise NotImplementedError("abstract") - def visitor_walk_recursive(self, instbox, visitor, optimizer): + def visitor_walk_recursive(self, instbox, visitor): instbox = instbox.get_box_replacement() if visitor.already_seen_virtual(instbox): return - return self._visitor_walk_recursive(instbox, visitor, optimizer) + return self._visitor_walk_recursive(instbox, visitor) class AbstractStructPtrInfo(AbstractVirtualPtrInfo): @@ -181,7 +182,7 @@ self.descr = descr self._fields = [None] * len(descr.get_all_fielddescrs()) if index >= len(self._fields): - self.descr = descr # a more precise descr + self.descr = descr # a more precise descr # we found out a subclass with more fields extra_len = len(descr.get_all_fielddescrs()) - len(self._fields) self._fields = self._fields + [None] * extra_len @@ -218,7 +219,7 @@ for i, fielddescr in enumerate(descr.get_all_fielddescrs()): fld = self._fields[i] if fld is not None: - subbox = optforce.force_box(fld) + subbox = optforce.optimizer.force_box(fld) setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox], descr=fielddescr) self._fields[i] = None @@ -226,31 +227,30 @@ def _force_at_the_end_of_preamble(self, op, optforce, rec): if self._fields is None: - return optforce.get_box_replacement(op) + return get_box_replacement(op) if self in rec: - return optforce.get_box_replacement(op) + return get_box_replacement(op) rec[self] = None for i, fldbox in enumerate(self._fields): if fldbox is not None: - info = optforce.getptrinfo(fldbox) + info = getptrinfo(fldbox) if info is not None: fldbox = info.force_at_the_end_of_preamble(fldbox, optforce, rec) self._fields[i] = fldbox return op - def _visitor_walk_recursive(self, instbox, visitor, optimizer): + def _visitor_walk_recursive(self, instbox, visitor): lst = self.descr.get_all_fielddescrs() assert self.is_virtual() - visitor.register_virtual_fields(instbox, - [optimizer.get_box_replacement(box) - for box in self._fields]) + visitor.register_virtual_fields( + instbox, [get_box_replacement(box) for box in self._fields]) for i in range(len(lst)): op = self._fields[i] if op: - fieldinfo = optimizer.getptrinfo(op) + fieldinfo = getptrinfo(op) if fieldinfo and fieldinfo.is_virtual(): - fieldinfo.visitor_walk_recursive(op, visitor, optimizer) + fieldinfo.visitor_walk_recursive(op, visitor) def produce_short_preamble_ops(self, structbox, fielddescr, index, optimizer, shortboxes): @@ -259,7 +259,7 @@ if fielddescr.get_index() >= len(self._fields): # we don't know about this item return - op = optimizer.get_box_replacement(self._fields[fielddescr.get_index()]) + op = get_box_replacement(self._fields[fielddescr.get_index()]) if op is None: # XXX same bug as in serialize_opt: # op should never be None, because that's an invariant violation in @@ -290,7 +290,7 @@ if op.is_constant(): pass # it is a constant value: ok else: - fieldinfo = optimizer.getptrinfo(op) + fieldinfo = getptrinfo(op) if fieldinfo and fieldinfo.is_virtual(): # recursive check if memo is None: @@ -302,12 +302,12 @@ return False # not a constant at all return True - def _force_elements_immutable(self, descr, constptr, optforce): + def _force_elements_immutable(self, descr, constptr, optimizer): for i, fielddescr in enumerate(descr.get_all_fielddescrs()): fld = self._fields[i] - subbox = optforce.force_box(fld) + subbox = optimizer.force_box(fld) assert isinstance(subbox, Const) - execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC, + execute(optimizer.cpu, None, rop.SETFIELD_GC, fielddescr, constptr, subbox) class InstancePtrInfo(AbstractStructPtrInfo): @@ -435,8 +435,8 @@ [op, ConstInt(offset), itembox], descr=descr) optforce.emit_extra(setfield_op) - def _visitor_walk_recursive(self, op, visitor, optimizer): - itemboxes = [optimizer.get_box_replacement(box) + def _visitor_walk_recursive(self, op, visitor): + itemboxes = [get_box_replacement(box) for box in self._get_buffer().values] visitor.register_virtual_fields(op, itemboxes) # there can be no virtuals stored in raw buffer @@ -465,21 +465,21 @@ return self.parent is not None def getitem_raw(self, offset, itemsize, descr): - return self.parent.getitem_raw(self.offset+offset, itemsize, descr) + return self.parent.getitem_raw(self.offset + offset, itemsize, descr) def setitem_raw(self, offset, itemsize, descr, itemop): - self.parent.setitem_raw(self.offset+offset, itemsize, descr, itemop) + self.parent.setitem_raw(self.offset + offset, itemsize, descr, itemop) def _force_elements(self, op, optforce, descr): if self.parent.is_virtual(): self.parent._force_elements(op, optforce, descr) self.parent = None - def _visitor_walk_recursive(self, op, visitor, optimizer): - source_op = optimizer.get_box_replacement(op.getarg(0)) + def _visitor_walk_recursive(self, op, visitor): + source_op = get_box_replacement(op.getarg(0)) visitor.register_virtual_fields(op, [source_op]) if self.parent.is_virtual(): - self.parent.visitor_walk_recursive(source_op, visitor, optimizer) + self.parent.visitor_walk_recursive(source_op, visitor) @specialize.argtype(1) def visitor_dispatch_virtual_type(self, visitor): @@ -534,7 +534,7 @@ def _force_elements(self, op, optforce, descr): # XXX descr = op.getdescr() - const = optforce.new_const_item(self.descr) + const = optforce.optimizer.new_const_item(self.descr) for i in range(self.length): item = self._items[i] if item is None: @@ -543,7 +543,7 @@ # clear the item so we don't know what's there self._items[i] = None continue - subbox = optforce.force_box(item) + subbox = optforce.optimizer.force_box(item) setop = ResOperation(rop.SETARRAYITEM_GC, [op, ConstInt(i), subbox], descr=descr) @@ -570,17 +570,15 @@ def getlength(self): return self.length - def _visitor_walk_recursive(self, instbox, visitor, optimizer): - itemops = [optimizer.get_box_replacement(item) - for item in self._items] + def _visitor_walk_recursive(self, instbox, visitor): + itemops = [get_box_replacement(item) for item in self._items] visitor.register_virtual_fields(instbox, itemops) for i in range(self.getlength()): itemop = self._items[i] - if (itemop is not None and - not isinstance(itemop, Const)): - ptrinfo = optimizer.getptrinfo(itemop) + if (itemop is not None and not isinstance(itemop, Const)): + ptrinfo = getptrinfo(itemop) if ptrinfo and ptrinfo.is_virtual(): - ptrinfo.visitor_walk_recursive(itemop, visitor, optimizer) + ptrinfo.visitor_walk_recursive(itemop, visitor) @specialize.argtype(1) def visitor_dispatch_virtual_type(self, visitor): @@ -596,7 +594,7 @@ item = self._items[index] if item is not None: # see comment in AbstractStructPtrInfo.produce_short_preamble_ops - op = optimizer.get_box_replacement(item) + op = get_box_replacement(item) opnum = OpHelpers.getarrayitem_for_descr(descr) getarrayitem_op = ResOperation(opnum, [structbox, ConstInt(index)], descr=descr) @@ -604,13 +602,13 @@ def _force_at_the_end_of_preamble(self, op, optforce, rec): if self._items is None: - return optforce.get_box_replacement(op) + return get_box_replacement(op) if self in rec: - return optforce.get_box_replacement(op) + return get_box_replacement(op) rec[self] = None for i, fldbox in enumerate(self._items): if fldbox is not None: - info = optforce.getptrinfo(fldbox) + info = getptrinfo(fldbox) if info is not None: fldbox = info.force_at_the_end_of_preamble(fldbox, optforce, rec) @@ -640,7 +638,7 @@ def _compute_index(self, index, fielddescr): all_fdescrs = fielddescr.get_arraydescr().get_all_fielddescrs() if all_fdescrs is None: - return 0 # annotation hack + return 0 # annotation hack one_size = len(all_fdescrs) return index * one_size + fielddescr.get_field_descr().get_index() @@ -659,7 +657,7 @@ for fielddescr in fielddescrs: fld = self._items[i] if fld is not None: - subbox = optforce.force_box(fld) + subbox = optforce.optimizer.force_box(fld) setfieldop = ResOperation(rop.SETINTERIORFIELD_GC, [op, ConstInt(index), subbox], descr=fielddescr) @@ -668,21 +666,18 @@ # if it does, we would need a fix here i += 1 - def _visitor_walk_recursive(self, instbox, visitor, optimizer): - itemops = [optimizer.get_box_replacement(item) - for item in self._items] + def _visitor_walk_recursive(self, instbox, visitor): + itemops = [get_box_replacement(item) for item in self._items] visitor.register_virtual_fields(instbox, itemops) fielddescrs = self.descr.get_all_fielddescrs() i = 0 for index in range(self.getlength()): for fielddescr in fielddescrs: itemop = self._items[i] - if (itemop is not None and - not isinstance(itemop, Const)): - ptrinfo = optimizer.getptrinfo(itemop) + if (itemop is not None and not isinstance(itemop, Const)): + ptrinfo = getptrinfo(itemop) if ptrinfo and ptrinfo.is_virtual(): - ptrinfo.visitor_walk_recursive(itemop, visitor, - optimizer) + ptrinfo.visitor_walk_recursive(itemop, visitor) i += 1 @specialize.argtype(1) @@ -705,7 +700,8 @@ def _get_info(self, descr, optheap): ref = self._const.getref_base() - if not ref: raise InvalidLoop # null protection + if not ref: + raise InvalidLoop # null protection info = optheap.const_infos.get(ref, None) if info is None: info = StructPtrInfo(descr) @@ -714,7 +710,8 @@ def _get_array_info(self, descr, optheap): ref = self._const.getref_base() - if not ref: raise InvalidLoop # null protection + if not ref: + raise InvalidLoop # null protection info = optheap.const_infos.get(ref, None) if info is None: info = ArrayPtrInfo(descr) @@ -780,8 +777,8 @@ return self._unpack_str(mode) def getlenbound(self, mode): - from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound,\ - IntLowerBound + from rpython.jit.metainterp.optimizeopt.intutils import ( + ConstIntBound, IntLowerBound) length = self.getstrlen1(mode) if length < 0: @@ -848,3 +845,36 @@ def make_guards(self, op, short, optimizer): short.append(ResOperation(rop.GUARD_VALUE, [op, self._const])) + + +def getrawptrinfo(op): + from rpython.jit.metainterp.optimizeopt.intutils import IntBound + assert op.type == 'i' + op = op.get_box_replacement() + assert op.type == 'i' + if isinstance(op, ConstInt): + return ConstPtrInfo(op) + fw = op.get_forwarded() + if isinstance(fw, IntBound): + return None + if fw is not None: + assert isinstance(fw, AbstractRawPtrInfo) + return fw + return None + +def getptrinfo(op): + if op.type == 'i': + return getrawptrinfo(op) + elif op.type == 'f': + return None + assert op.type == 'r' + op = get_box_replacement(op) + assert op.type == 'r' + if isinstance(op, ConstPtr): + return ConstPtrInfo(op) + fw = op.get_forwarded() + if fw is not None: + assert isinstance(fw, PtrInfo) + return fw + return None + diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -5,7 +5,9 @@ IntLowerBound, IntUpperBound, ConstIntBound) from rpython.jit.metainterp.optimizeopt.optimizer import (Optimization, CONST_1, CONST_0) -from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method +from rpython.jit.metainterp.optimizeopt.util import ( + make_dispatcher_method, get_box_replacement) +from .info import getptrinfo from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.optimizeopt import vstring from rpython.jit.codewriter.effectinfo import EffectInfo @@ -64,8 +66,8 @@ postprocess_GUARD_VALUE = _postprocess_guard_true_false_value def optimize_INT_OR_or_XOR(self, op): - v1 = self.get_box_replacement(op.getarg(0)) - v2 = self.get_box_replacement(op.getarg(1)) + v1 = get_box_replacement(op.getarg(0)) + v2 = get_box_replacement(op.getarg(1)) if v1 is v2: if op.getopnum() == rop.INT_OR: self.make_equal_to(op, v1) @@ -75,9 +77,9 @@ return self.emit(op) def postprocess_INT_OR_or_XOR(self, op): - v1 = self.get_box_replacement(op.getarg(0)) + v1 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(v1) - v2 = self.get_box_replacement(op.getarg(1)) + v2 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(v2) b = b1.or_bound(b2) self.getintbound(op).intersect(b) @@ -108,8 +110,8 @@ self.getintbound(op).intersect(b) def optimize_INT_ADD(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) if self.is_raw_ptr(arg1) or self.is_raw_ptr(arg2): return self.emit(op) v1 = self.getintbound(arg1) @@ -130,8 +132,8 @@ arg1 = self.optimizer.as_operation(arg1) if arg1 is not None: if arg1.getopnum() == rop.INT_ADD: - prod_arg1 = self.get_box_replacement(arg1.getarg(0)) - prod_arg2 = self.get_box_replacement(arg1.getarg(1)) + prod_arg1 = get_box_replacement(arg1.getarg(0)) + prod_arg2 = get_box_replacement(arg1.getarg(1)) prod_v1 = self.getintbound(prod_arg1) prod_v2 = self.getintbound(prod_arg2) @@ -196,9 +198,9 @@ return self.emit(op) def postprocess_INT_LSHIFT(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) + arg0 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(arg0) - arg1 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(arg1) r = self.getintbound(op) b = b1.lshift_bound(b2) @@ -290,8 +292,8 @@ r.intersect(resbound) def optimize_INT_SUB_OVF(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) - arg1 = self.get_box_replacement(op.getarg(1)) + arg0 = get_box_replacement(op.getarg(0)) + arg1 = get_box_replacement(op.getarg(1)) b0 = self.getintbound(arg0) b1 = self.getintbound(arg1) if arg0.same_box(arg1): @@ -304,8 +306,8 @@ return self.emit(op) def postprocess_INT_SUB_OVF(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) - arg1 = self.get_box_replacement(op.getarg(1)) + arg0 = get_box_replacement(op.getarg(0)) + arg1 = get_box_replacement(op.getarg(1)) b0 = self.getintbound(arg0) b1 = self.getintbound(arg1) resbound = b0.sub_bound(b1) @@ -329,8 +331,8 @@ r.intersect(resbound) def optimize_INT_LT(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b1.known_lt(b2): @@ -341,8 +343,8 @@ return self.emit(op) def optimize_INT_GT(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b1.known_gt(b2): @@ -353,8 +355,8 @@ return self.emit(op) def optimize_INT_LE(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b1.known_le(b2) or arg1 is arg2: @@ -365,8 +367,8 @@ return self.emit(op) def optimize_INT_GE(self, op): - arg1 = self.get_box_replacement(op.getarg(0)) - arg2 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(0)) + arg2 = get_box_replacement(op.getarg(1)) b1 = self.getintbound(arg1) b2 = self.getintbound(arg2) if b1.known_ge(b2) or arg1 is arg2: @@ -377,9 +379,9 @@ return self.emit(op) def optimize_INT_EQ(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) + arg0 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(arg0) - arg1 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(arg1) if b1.known_gt(b2): self.make_constant_int(op, 0) @@ -391,9 +393,9 @@ return self.emit(op) def optimize_INT_NE(self, op): - arg0 = self.get_box_replacement(op.getarg(0)) + arg0 = get_box_replacement(op.getarg(0)) b1 = self.getintbound(arg0) - arg1 = self.get_box_replacement(op.getarg(1)) + arg1 = get_box_replacement(op.getarg(1)) b2 = self.getintbound(arg1) if b1.known_gt(b2): self.make_constant_int(op, 1) @@ -442,7 +444,7 @@ def postprocess_STRLEN(self, op): self.make_nonnull_str(op.getarg(0), vstring.mode_string) - array = self.getptrinfo(op.getarg(0)) + array = getptrinfo(op.getarg(0)) self.optimizer.setintbound(op, array.getlenbound(vstring.mode_string)) def optimize_UNICODELEN(self, op): @@ -450,7 +452,7 @@ def postprocess_UNICODELEN(self, op): self.make_nonnull_str(op.getarg(0), vstring.mode_unicode) - array = self.getptrinfo(op.getarg(0)) + array = getptrinfo(op.getarg(0)) self.optimizer.setintbound(op, array.getlenbound(vstring.mode_unicode)) def optimize_STRGETITEM(self, op): @@ -458,7 +460,7 @@ def postprocess_STRGETITEM(self, op): v1 = self.getintbound(op) - v2 = self.getptrinfo(op.getarg(0)) + v2 = getptrinfo(op.getarg(0)) intbound = self.getintbound(op.getarg(1)) if (intbound.has_lower and v2 is not None and v2.getlenbound(vstring.mode_string) is not None): @@ -523,7 +525,7 @@ def postprocess_UNICODEGETITEM(self, op): b1 = self.getintbound(op) b1.make_ge(IntLowerBound(0)) - v2 = self.getptrinfo(op.getarg(0)) + v2 = getptrinfo(op.getarg(0)) intbound = self.getintbound(op.getarg(1)) if (intbound.has_lower and v2 is not None and v2.getlenbound(vstring.mode_unicode) is not None): diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -1,16 +1,19 @@ from rpython.jit.metainterp import jitprof, resume, compile from rpython.jit.metainterp.executor import execute_nonspec_const from rpython.jit.metainterp.history import ( - Const, ConstInt, ConstPtr, CONST_NULL, new_ref_dict) -from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\ - ConstIntBound, MININT, MAXINT, IntUnbounded -from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method -from rpython.jit.metainterp.resoperation import rop, AbstractResOp, GuardResOp,\ - OpHelpers + Const, ConstInt, CONST_NULL, new_ref_dict) +from rpython.jit.metainterp.optimizeopt.intutils import ( + IntBound, ConstIntBound, MININT, MAXINT, IntUnbounded) +from rpython.jit.metainterp.optimizeopt.util import ( + make_dispatcher_method, get_box_replacement) +from rpython.jit.metainterp.optimizeopt.bridgeopt import ( + deserialize_optimizer_knowledge) +from rpython.jit.metainterp.resoperation import ( + rop, AbstractResOp, GuardResOp, OpHelpers) +from .info import getrawptrinfo, getptrinfo from rpython.jit.metainterp.optimizeopt import info from rpython.jit.metainterp.optimize import InvalidLoop from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.debug import debug_print from rpython.rtyper import rclass from rpython.rtyper.lltypesystem import llmemory from rpython.jit.metainterp.optimize import SpeculativeError @@ -57,9 +60,6 @@ def __init__(self): pass # make rpython happy - def send_extra_operation(self, op, opt=None): - self.optimizer.send_extra_operation(op, opt) - From pypy.commits at gmail.com Wed Jun 5 06:28:50 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 05 Jun 2019 03:28:50 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: kill hopefully dead code Message-ID: <5cf79962.1c69fb81.2de25.bfdc@mx.google.com> Author: fijal Branch: copystrcontents-in-rewrite Changeset: r96748:6fe2a86cdc00 Date: 2019-06-04 13:34 +0200 http://bitbucket.org/pypy/pypy/changeset/6fe2a86cdc00/ Log: kill hopefully dead code diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1233,79 +1233,6 @@ self.assembler.load_effective_addr(iloc, op.getarg(2).getint(), op.getarg(3).getint(), res, ploc) - def consider_copystrcontent(self, op): - self._consider_copystrcontent(op, is_unicode=False) - - def consider_copyunicodecontent(self, op): - self._consider_copystrcontent(op, is_unicode=True) - - def _consider_copystrcontent(self, op, is_unicode): - # compute the source address - args = op.getarglist() - base_loc = self.rm.make_sure_var_in_reg(args[0], args) - ofs_loc = self.rm.make_sure_var_in_reg(args[2], args) - assert args[0] is not args[1] # forbidden case of aliasing - srcaddr_box = TempVar() - forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = self.rm.force_allocate_reg(srcaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, - is_unicode=is_unicode) - # compute the destination address - base_loc = self.rm.make_sure_var_in_reg(args[1], forbidden_vars) - ofs_loc = self.rm.make_sure_var_in_reg(args[3], forbidden_vars) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempVar() - dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, - is_unicode=is_unicode) - # compute the length in bytes - length_box = args[4] - length_loc = self.loc(length_box) - if is_unicode: - forbidden_vars = [srcaddr_box, dstaddr_box] - bytes_box = TempVar() - bytes_loc = self.rm.force_allocate_reg(bytes_box, forbidden_vars) - scale = self._get_unicode_item_scale() - if not (isinstance(length_loc, ImmedLoc) or - isinstance(length_loc, RegLoc)): - self.assembler.mov(length_loc, bytes_loc) - length_loc = bytes_loc - self.assembler.load_effective_addr(length_loc, 0, scale, bytes_loc) - length_box = bytes_box - length_loc = bytes_loc - # call memcpy() - self.rm.before_call() - self.xrm.before_call() - self.assembler.simple_call_no_collect(imm(self.assembler.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) - self.rm.possibly_free_var(length_box) - self.rm.possibly_free_var(dstaddr_box) - self.rm.possibly_free_var(srcaddr_box) - - def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): - if is_unicode: - ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - scale = self._get_unicode_item_scale() - else: - ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.translate_support_code) - assert itemsize == 1 - ofs_items -= 1 # for the extra null character - scale = 0 - self.assembler.load_effective_addr(ofsloc, ofs_items, scale, - resloc, baseloc) - - def _get_unicode_item_scale(self): - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") - def _consider_math_read_timestamp(self, op): # hint: try to move unrelated registers away from eax and edx now self.rm.spill_or_move_registers_before_call([eax, edx]) From pypy.commits at gmail.com Wed Jun 5 06:28:52 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 05 Jun 2019 03:28:52 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: maybe fix ARM Message-ID: <5cf79964.1c69fb81.f2a5.bae5@mx.google.com> Author: fijal Branch: copystrcontents-in-rewrite Changeset: r96749:805a83055a7d Date: 2019-06-04 13:45 +0200 http://bitbucket.org/pypy/pypy/changeset/805a83055a7d/ Log: maybe fix ARM diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -834,72 +834,8 @@ else: assert 0 - #from ../x86/regalloc.py:928 ff. - def emit_op_copystrcontent(self, op, arglocs, regalloc, fcond): - assert len(arglocs) == 0 - self._emit_copystrcontent(op, regalloc, fcond, is_unicode=False) - return fcond - - def emit_op_copyunicodecontent(self, op, arglocs, regalloc, fcond): - assert len(arglocs) == 0 - self._emit_copystrcontent(op, regalloc, fcond, is_unicode=True) - return fcond - - def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): - # compute the source address - args = op.getarglist() - base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) - ofs_loc = regalloc.rm.make_sure_var_in_reg(args[2], args) - assert args[0] is not args[1] # forbidden case of aliasing - srcaddr_box = TempVar() - forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = regalloc.rm.force_allocate_reg(srcaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, - is_unicode=is_unicode) - # compute the destination address - base_loc = regalloc.rm.make_sure_var_in_reg(args[1], forbidden_vars) - ofs_loc = regalloc.rm.make_sure_var_in_reg(args[3], forbidden_vars) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempVar() - dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, - is_unicode=is_unicode) - # compute the length in bytes - length_box = args[4] - length_loc = regalloc.loc(length_box) - if is_unicode: - forbidden_vars = [srcaddr_box, dstaddr_box] - bytes_box = TempVar() - bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, forbidden_vars) - scale = self._get_unicode_item_scale() - if not length_loc.is_core_reg(): - self.regalloc_mov(length_loc, bytes_loc) - length_loc = bytes_loc - assert length_loc.is_core_reg() - self.mc.MOV_ri(r.ip.value, 1 << scale) - self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) - length_box = bytes_box - length_loc = bytes_loc - # call memcpy() - regalloc.before_call() - self.simple_call_no_collect(imm(self.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) - regalloc.rm.possibly_free_var(length_box) - regalloc.rm.possibly_free_var(dstaddr_box) - regalloc.rm.possibly_free_var(srcaddr_box) - - def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): - if is_unicode: - ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - scale = self._get_unicode_item_scale() - else: - ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - ofs_items -= 1 # for the extra null character - scale = 0 - self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items) + def emit_op_load_effective_address(self, op, arglocs, regalloc, fcond): + self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[2], arglocs[3]) # result = base_loc + (scaled_loc << scale) + static_offset def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0): @@ -915,16 +851,6 @@ self.mc.ADD_rr(result.value, base_loc.value, scaled_loc.value) self.mc.ADD_ri(result.value, result.value, static_offset) - def _get_unicode_item_scale(self): - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") - def store_force_descr(self, op, fail_locs, frame_depth): pos = self.mc.currpos() guard_token = self.build_guard_token(op, frame_depth, fail_locs, pos, c.AL) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -873,8 +873,6 @@ prepare_op_gc_load_indexed_r = _prepare_op_gc_load_indexed prepare_op_gc_load_indexed_f = _prepare_op_gc_load_indexed - prepare_op_copystrcontent = void - prepare_op_copyunicodecontent = void prepare_op_zero_array = void def _prepare_op_same_as(self, op, fcond): @@ -899,6 +897,13 @@ resloc = self.force_allocate_reg(op) return [resloc] + def prepare_op_load_effective_address(self, op, fcond): + args = op.getarglist() + arg0 = self.make_sure_var_in_reg(args[0], args) + arg1 = self.make_sure_var_in_reg(args[1], args) + res = self.force_allocate_reg(op) + return [arg0, arg1, args[2], args[3], res] + def prepare_op_call_malloc_nursery(self, op, fcond): size_box = op.getarg(0) assert isinstance(size_box, ConstInt) From pypy.commits at gmail.com Wed Jun 5 06:28:54 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 05 Jun 2019 03:28:54 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: start cargo culting PPC support Message-ID: <5cf79966.1c69fb81.65583.b4b4@mx.google.com> Author: fijal Branch: copystrcontents-in-rewrite Changeset: r96750:07999e1e391f Date: 2019-06-05 12:27 +0200 http://bitbucket.org/pypy/pypy/changeset/07999e1e391f/ Log: start cargo culting PPC support diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -802,6 +802,14 @@ temp_loc = r.SCRATCH2 return [base_loc, temp_loc] + def prepare_load_effective_address(self, op): + arg0 = self.ensure_reg(op.getarg(0)) + arg1 = self.ensure_reg(op.getarg(1)) + arg2 = self.ensure_reg_or_any_imm(op.getarg(2)) + arg3 = self.ensure_reg_or_any_imm(op.getarg(3)) + resloc = self.force_allocate_reg(op) + return [arg0, arg1, arg2, arg3, resloc] + def prepare_copystrcontent(self, op): src_ptr_loc = self.ensure_reg(op.getarg(0)) dst_ptr_loc = self.ensure_reg(op.getarg(1)) From pypy.commits at gmail.com Wed Jun 5 06:32:39 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 05 Jun 2019 03:32:39 -0700 (PDT) Subject: [pypy-commit] pypy arm64: merge copystrcontents in rewrite branch Message-ID: <5cf79a47.1c69fb81.d515a.81d6@mx.google.com> Author: fijal Branch: arm64 Changeset: r96751:44fd26cfc9c9 Date: 2019-06-05 12:31 +0200 http://bitbucket.org/pypy/pypy/changeset/44fd26cfc9c9/ Log: merge copystrcontents in rewrite branch diff too long, truncating to 2000 out of 251507 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -4,8 +4,10 @@ *~ .*.swp .idea +.mypy_cache .project .pydevproject +.vscode __pycache__ .cache/ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -10,10 +10,6 @@ 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0 9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1 -fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0 -fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0 -e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0 -e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0 295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0 f3ad1e1e1d6215e20d34bb65ab85ff9188c9f559 release-2.6.1 850edf14b2c75573720f59e95767335fb1affe55 release-4.0.0 @@ -24,17 +20,10 @@ b0a649e90b6642251fb4a765fe5b27a97b1319a9 release-5.1.1 80ef432a32d9baa4b3c5a54c215e8ebe499f6374 release-5.1.2 40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 -40497617ae91caa1a394d8be6f9cd2de31cb0628 release-pypy3.3-v5.2 c09c19272c990a0611b17569a0085ad1ab00c8ff release-pypy2.7-v5.3 7e8df3df96417c16c2d55b41352ec82c9c69c978 release-pypy2.7-v5.3.1 -68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 -68bb3510d8212ae9efb687e12e58c09d29e74f87 release-pypy2.7-v5.4.0 77392ad263504df011ccfcabf6a62e21d04086d0 release-pypy2.7-v5.4.0 -050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 -050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1 -4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1 -4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1 d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0 e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1 @@ -45,16 +34,20 @@ 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1 c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0 a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0 -03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 -03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 -a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 -a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 -0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 -0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0 fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0 +9112c8071614108b1042bfef0713915107004d62 release-pypy2.7-v7.0.0 +1f86f25937b6ae6c8b25236c35228fac587678bf release-pypy3.5-v7.0.0 +c8805ee6d7846ca2722b106eeaa2f128c699aba3 release-pypy2.7-v7.0.0 +928a4f70d3de7d17449456946154c5da6e600162 release-pypy3.5-v7.0.0 +fb40f7a5524c77b80e6c468e087d621610137261 release-pypy3.6-v7.0.0 +990cef41fe11e5d46b019a46aa956ff46ea1a234 release-pypy2.7-v7.1.0 +de061d87e39c7df4e436974096d7982c676a859d release-pypy3.6-v7.1.0 +784b254d669919c872a505b807db8462b6140973 release-pypy3.6-v7.1.1 +8cdda8b8cdb8ff29d9e620cccd6c5edd2f2a23ec release-pypy2.7-v7.1.1 + diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -40,16 +40,16 @@ Armin Rigo Maciej Fijalkowski Carl Friedrich Bolz-Tereick + Antonio Cuni Amaury Forgeot d'Arc - Antonio Cuni Matti Picus Samuele Pedroni Ronan Lamy Alex Gaynor Philip Jenvey + Richard Plangger Brian Kearns - Richard Plangger - Michael Hudson + Michael Hudson-Doyle Manuel Jacob David Schneider Holger Krekel @@ -59,8 +59,8 @@ Anders Chrigstrom Wim Lavrijsen Eric van Riet Paap + Remi Meier Richard Emslie - Remi Meier Alexander Schremmer Dan Villiom Podlaski Christiansen Lukas Diekmann @@ -70,10 +70,10 @@ Niklaus Haldimann Camillo Bruni Laura Creighton - Romain Guillebert Toon Verwaest Leonardo Santagada Seo Sanghyeon + Romain Guillebert Ronny Pfannschmidt Justin Peel Raffael Tfirst @@ -110,19 +110,21 @@ Devin Jeanpierre Bob Ippolito Bruno Gola + Andrew Lawrence David Malcolm Squeaky Edd Barrett Timo Paulssen + Laurence Tratt Marius Gedminas Nicolas Truessel Alexandre Fayolle Simon Burton Martin Matusiak - Laurence Tratt Wenzhu Man Konstantin Lopuhin John Witulski + Stefan Beyer Jeremy Thurgood Greg Price Ivan Sichmann Freitas @@ -134,8 +136,8 @@ Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov + William Leslie Paweł Piotr Przeradowski - William Leslie marky1991 Ilya Osadchiy Tobias Oberstein @@ -144,13 +146,14 @@ Taavi Burns Adrian Kuhn tav + Stian Andreassen Georg Brandl Joannah Nanjekye Bert Freudenberg - Stian Andreassen Wanja Saatkamp Mike Blume Gerald Klix + Julian Berman Oscar Nierstrasz Rami Chowdhury Stefan H. Muller @@ -163,6 +166,7 @@ Vasily Kuznetsov Preston Timmons David Ripton + Pieter Zieschang Dusty Phillips Lukas Renggli Guenter Jantzen @@ -172,10 +176,12 @@ Anton Gulenko Sergey Matyunin Andrew Chambers + Łukasz Langa Nicolas Chauvat Andrew Durdin Ben Young Michael Schneider + Yusuke Tsutsumi Nicholas Riley Jason Chu Igor Trindade Oliveira @@ -187,7 +193,6 @@ Mariano Anaya anatoly techtonik Karl Bartel - Stefan Beyer Gabriel Lavoie Jared Grubb Alecsandru Patrascu @@ -198,7 +203,6 @@ Victor Stinner Andrews Medina Aaron Iles - p_zieschang at yahoo.de Toby Watson Daniel Patrick Stuart Williams @@ -210,6 +214,7 @@ Mikael Schönenberg Stanislaw Halik Mihnea Saracin + Matt Jackson Berkin Ilbeyi Gasper Zejn Faye Zhao @@ -217,12 +222,14 @@ Anders Qvist Corbin Simpson Chirag Jadwani + Pauli Virtanen Jonathan David Riehl Beatrice During Alex Perry Robert Zaremba Alan McIntyre Alexander Sedov + David C Ellis Vaibhav Sood Reuben Cummings Attila Gobi @@ -242,12 +249,12 @@ Arjun Naik Aaron Gallagher Alexis Daboville - Pieter Zieschang Karl Ramm Lukas Vacek Omer Katz Jacek Generowicz Tomasz Dziopa + Lin Cheng Sylvain Thenault Jakub Stasiak Andrew Dalke @@ -270,12 +277,15 @@ Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez + Antoine Dupre Kristjan Valur Jonsson Lucio Torre Richard Lancaster Dan Buch Lene Wagner Tomo Cocoa + Miro Hrončok + Anthony Sottile David Lievens Neil Blakey-Milner Henrik Vendelbo @@ -294,6 +304,7 @@ Aaron Tubbs Vasantha Ganesh K Jason Michalski + Radu Ciorba Markus Holtermann Andrew Thompson Yusei Tahara @@ -301,28 +312,25 @@ Fabio Niephaus Akira Li Gustavo Niemeyer - Rafał Gałczyński + Nate Bragg Lucas Stadler roberto at goyle + Carl Bordum Hansen Matt Bogosian Yury V. Zaytsev florinpapa Anders Sigfridsson - Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert Anna Katrina Dominguez Kim Jin Su Amber Brown - Miro Hrončok - Anthony Sottile - Nate Bragg + Andrew Stepanov + Rafał Gałczyński Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle - Julian Berman - Michael Hudson-Doyle Stephan Busemann Dan Colish timo @@ -332,6 +340,7 @@ halgari Jim Baker Chris Lambacher + John Aldis coolbutuseless at gmail.com Mike Bayer Rodrigo Araújo @@ -340,6 +349,7 @@ OlivierBlanvillain Jonas Pfannschmidt Zearin + Johan Forsberg Andrey Churin Dan Crosta reubano at gmail.com @@ -349,8 +359,9 @@ Steve Papanik Eli Stevens Boglarka Vezer - gabrielg + gabrielg at ec2-54-146-239-158.compute-1.amazonaws.com PavloKapyshin + Hervé Beraud Tomer Chachamu Christopher Groskopf Asmo Soinio @@ -364,7 +375,6 @@ Michael Chermside Anna Ravencroft remarkablerocket - Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -384,15 +394,17 @@ Zooko Wilcox-O Hearn James Lan jiaaro + Evgenii Gorinov Markus Unterwaditzer Kristoffer Kleine Graham Markall Dan Loewenherz werat - Andrew Stepanov + Filip Salomonsson Niclas Olofsson Chris Pressey Tobias Diaz + Paul Graydon Nikolaos-Digenis Karagiannis Kurt Griffiths Ben Mather diff --git a/TODO b/TODO new file mode 100644 --- /dev/null +++ b/TODO @@ -0,0 +1,4 @@ +* find a better way to run "find" without creating the index storage, + if one is not already readily available (understand cost now, improve after merge) +* improve performance of splitlines +* think about cost of utf8 list strategy (Armin and CF) diff --git a/extra_tests/cffi_tests/cffi0/backend_tests.py b/extra_tests/cffi_tests/cffi0/backend_tests.py --- a/extra_tests/cffi_tests/cffi0/backend_tests.py +++ b/extra_tests/cffi_tests/cffi0/backend_tests.py @@ -1,5 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py +import pytest import platform import sys, ctypes from cffi import FFI, CDefError, FFIError, VerificationMissing @@ -113,10 +114,14 @@ p[9] = 43 assert p[0] == 42 assert p[9] == 43 - py.test.raises(IndexError, "p[10]") - py.test.raises(IndexError, "p[10] = 44") - py.test.raises(IndexError, "p[-1]") - py.test.raises(IndexError, "p[-1] = 44") + with pytest.raises(IndexError): + p[10] + with pytest.raises(IndexError): + p[10] = 44 + with pytest.raises(IndexError): + p[-1] + with pytest.raises(IndexError): + p[-1] = 44 def test_new_array_args(self): ffi = FFI(backend=self.Backend()) @@ -141,18 +146,21 @@ ffi = FFI(backend=self.Backend()) p = ffi.new("int[]", 10) # a single integer is the length assert p[9] == 0 - py.test.raises(IndexError, "p[10]") + with pytest.raises(IndexError): + p[10] # py.test.raises(TypeError, ffi.new, "int[]") # p = ffi.new("int[]", [-6, -7]) # a list is all the items, like C assert p[0] == -6 assert p[1] == -7 - py.test.raises(IndexError, "p[2]") + with pytest.raises(IndexError): + p[2] assert repr(p) == "" % (2*SIZE_OF_INT) # p = ffi.new("int[]", 0) - py.test.raises(IndexError, "p[0]") + with pytest.raises(IndexError): + p[0] py.test.raises(ValueError, ffi.new, "int[]", -1) assert repr(p) == "" @@ -260,7 +268,8 @@ p[2][3] = 33 assert p[0][0] == 10 assert p[2][3] == 33 - py.test.raises(IndexError, "p[1][-1]") + with pytest.raises(IndexError): + p[1][-1] def test_constructor_array_of_array(self): ffi = FFI(backend=self.Backend()) @@ -387,7 +396,8 @@ n = ffi.new("int*", 99) p = ffi.new("int*[]", [n]) assert p[0][0] == 99 - py.test.raises(TypeError, "p[0] = None") + with pytest.raises(TypeError): + p[0] = None p[0] = ffi.NULL assert p[0] == ffi.NULL @@ -423,13 +433,15 @@ assert s.a == s.b == s.c == 0 s.b = -23 assert s.b == -23 - py.test.raises(OverflowError, "s.b = 32768") + with pytest.raises(OverflowError): + s.b = 32768 # s = ffi.new("struct foo*", [-2, -3]) assert s.a == -2 assert s.b == -3 assert s.c == 0 - py.test.raises((AttributeError, TypeError), "del s.a") + with pytest.raises((AttributeError, TypeError)): + del s.a assert repr(s) == "" % ( SIZE_OF_INT + 2 * SIZE_OF_SHORT) # @@ -451,8 +463,10 @@ assert s[0].a == s[0].b == s[0].c == 0 s[0].b = -23 assert s[0].b == s.b == -23 - py.test.raises(OverflowError, "s[0].b = -32769") - py.test.raises(IndexError, "s[1]") + with pytest.raises(OverflowError): + s[0].b = -32769 + with pytest.raises(IndexError): + s[1] def test_struct_opaque(self): ffi = FFI(backend=self.Backend()) @@ -512,11 +526,13 @@ u.b = -23 assert u.b == -23 assert u.a != 0 - py.test.raises(OverflowError, "u.b = 32768") + with pytest.raises(OverflowError): + u.b = 32768 # u = ffi.new("union foo*", [-2]) assert u.a == -2 - py.test.raises((AttributeError, TypeError), "del u.a") + with pytest.raises((AttributeError, TypeError)): + del u.a assert repr(u) == "" % SIZE_OF_INT def test_union_opaque(self): @@ -592,7 +608,8 @@ p[3] = b'\x00' assert ffi.string(p) == b"hel" assert ffi.string(p, 2) == b"he" - py.test.raises(IndexError, "p[7] = b'X'") + with pytest.raises(IndexError): + p[7] = b'X' # a = ffi.new("char[]", b"hello\x00world") assert len(a) == 12 @@ -616,7 +633,8 @@ p[3] = u+'\x00' assert ffi.string(p) == u+"hel" assert ffi.string(p, 123) == u+"hel" - py.test.raises(IndexError, "p[7] = u+'X'") + with pytest.raises(IndexError): + p[7] = u+'X' # a = ffi.new("wchar_t[]", u+"hello\x00world") assert len(a) == 12 @@ -634,7 +652,8 @@ s = ffi.new("struct foo*", [t]) assert type(s.name) not in (bytes, str, unicode) assert ffi.string(s.name) == b"testing" - py.test.raises(TypeError, "s.name = None") + with pytest.raises(TypeError): + s.name = None s.name = ffi.NULL assert s.name == ffi.NULL @@ -658,18 +677,21 @@ a = ffi.new("int[]", [10, 11, 12]) p = ffi.new("void **", a) vp = p[0] - py.test.raises(TypeError, "vp[0]") + with pytest.raises(TypeError): + vp[0] py.test.raises(TypeError, ffi.new, "short **", a) # ffi.cdef("struct foo { void *p; int *q; short *r; };") s = ffi.new("struct foo *") s.p = a # works s.q = a # works - py.test.raises(TypeError, "s.r = a") # fails + with pytest.raises(TypeError): + s.r = a # fails b = ffi.cast("int *", a) s.p = b # works s.q = b # works - py.test.raises(TypeError, "s.r = b") # fails + with pytest.raises(TypeError): + s.r = b # fails def test_functionptr_simple(self): ffi = FFI(backend=self.Backend()) @@ -688,7 +710,8 @@ q = ffi.new("int(**)(int)", p) assert repr(q) == "" % ( SIZE_OF_PTR) - py.test.raises(TypeError, "q(43)") + with pytest.raises(TypeError): + q(43) res = q[0](43) assert res == 44 q = ffi.cast("int(*)(int)", p) @@ -913,10 +936,14 @@ assert s.e == 4294967295 assert s[0].e == 4294967295 s.e = s.e - py.test.raises(TypeError, "s.e = 'B'") - py.test.raises(TypeError, "s.e = '2'") - py.test.raises(TypeError, "s.e = '#2'") - py.test.raises(TypeError, "s.e = '#7'") + with pytest.raises(TypeError): + s.e = 'B' + with pytest.raises(TypeError): + s.e = '2' + with pytest.raises(TypeError): + s.e = '#2' + with pytest.raises(TypeError): + s.e = '#7' def test_enum_non_contiguous(self): ffi = FFI(backend=self.Backend()) @@ -951,11 +978,14 @@ ffi = FFI(backend=self.Backend()) ffi.cdef("struct foo { int a, b; };") s = ffi.new("struct foo[1]") - py.test.raises(AttributeError, 's.b') - py.test.raises(AttributeError, 's.b = 412') + with pytest.raises(AttributeError): + s.b + with pytest.raises(AttributeError): + s.b = 412 s[0].b = 412 assert s[0].b == 412 - py.test.raises(IndexError, 's[1]') + with pytest.raises(IndexError): + s[1] def test_pointer_to_array(self): ffi = FFI(backend=self.Backend()) @@ -1012,17 +1042,23 @@ assert ffi.sizeof("struct foo") == 8 s = ffi.new("struct foo *") s.a = 511 - py.test.raises(OverflowError, "s.a = 512") - py.test.raises(OverflowError, "s[0].a = 512") + with pytest.raises(OverflowError): + s.a = 512 + with pytest.raises(OverflowError): + s[0].a = 512 assert s.a == 511 s.a = -512 - py.test.raises(OverflowError, "s.a = -513") - py.test.raises(OverflowError, "s[0].a = -513") + with pytest.raises(OverflowError): + s.a = -513 + with pytest.raises(OverflowError): + s[0].a = -513 assert s.a == -512 s.c = 3 assert s.c == 3 - py.test.raises(OverflowError, "s.c = 4") - py.test.raises(OverflowError, "s[0].c = 4") + with pytest.raises(OverflowError): + s.c = 4 + with pytest.raises(OverflowError): + s[0].c = 4 s.c = -4 assert s.c == -4 @@ -1280,7 +1316,8 @@ p = ffi.new("struct foo_s *", 10) # a single integer is the length assert p.len == 0 assert p.data[9] == 0 - py.test.raises(IndexError, "p.data[10]") + with pytest.raises(IndexError): + p.data[10] def test_ffi_typeof_getcname(self): ffi = FFI(backend=self.Backend()) diff --git a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py --- a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py +++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py @@ -130,6 +130,36 @@ alloc5 = ffi.new_allocator(myalloc5) py.test.raises(MemoryError, alloc5, "int[5]") + def test_new_struct_containing_struct_containing_array_varsize(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + struct foo_s { int len[100]; short data[]; }; + struct bar_s { int abc[100]; struct foo_s tail; }; + """) + # loop to try to detect heap overwrites, if the size allocated + # is too small + for i in range(1, 501, 100): + p = ffi.new("struct bar_s *", [[10], [[20], [3,4,5,6,7,8,9] * i]]) + assert p.abc[0] == 10 + assert p.tail.len[0] == 20 + assert p.tail.data[0] == 3 + assert p.tail.data[6] == 9 + assert p.tail.data[7 * i - 1] == 9 + + def test_bogus_struct_containing_struct_containing_array_varsize(self): + ffi = FFI(backend=self.Backend()) + ffi.cdef(""" + struct foo_s { signed char len; signed char data[]; }; + struct bar_s { struct foo_s foo; int bcd; }; + """) + p = ffi.new("struct bar_s *", [[123, [45, 56, 67, 78]], 9999999]) + assert p.foo.len == 123 + assert p.foo.data[0] == 45 + assert p.foo.data[1] == 56 + assert p.foo.data[2] == 67 + assert p.bcd == 9999999 + assert p.foo.data[3] != 78 # has been overwritten with 9999999 + class TestBitfield: def check(self, source, expected_ofs_y, expected_align, expected_size): @@ -269,12 +299,15 @@ def test_error_cases(self): ffi = FFI() - py.test.raises(TypeError, - 'ffi.cdef("struct s1 { float x:1; };"); ffi.new("struct s1 *")') - py.test.raises(TypeError, - 'ffi.cdef("struct s2 { char x:0; };"); ffi.new("struct s2 *")') - py.test.raises(TypeError, - 'ffi.cdef("struct s3 { char x:9; };"); ffi.new("struct s3 *")') + ffi.cdef("struct s1 { float x:1; };") + with pytest.raises(TypeError): + ffi.new("struct s1 *") + ffi.cdef("struct s2 { char x:0; };") + with pytest.raises(TypeError): + ffi.new("struct s2 *") + ffi.cdef("struct s3 { char x:9; };") + with pytest.raises(TypeError): + ffi.new("struct s3 *") def test_struct_with_typedef(self): ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi0/test_function.py b/extra_tests/cffi_tests/cffi0/test_function.py --- a/extra_tests/cffi_tests/cffi0/test_function.py +++ b/extra_tests/cffi_tests/cffi0/test_function.py @@ -1,5 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py +import pytest from cffi import FFI, CDefError import math, os, sys import ctypes.util @@ -46,14 +47,15 @@ assert x != math.sin(1.23) # rounding effects assert abs(x - math.sin(1.23)) < 1E-6 - def test_lround_no_return_value(self): + def test_getenv_no_return_value(self): # check that 'void'-returning functions work too ffi = FFI(backend=self.Backend()) ffi.cdef(""" - void lround(double x); + void getenv(char *); """) - m = ffi.dlopen(lib_m) - x = m.lround(1.23) + needs_dlopen_none() + m = ffi.dlopen(None) + x = m.getenv(b"FOO") assert x is None def test_dlopen_filename(self): @@ -90,7 +92,8 @@ """) m = ffi.dlopen(lib_m) assert m.FOOBAR == 42 - py.test.raises(NotImplementedError, "m.baz") + with pytest.raises(NotImplementedError): + m.baz def test_tlsalloc(self): if sys.platform != 'win32': diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py --- a/extra_tests/cffi_tests/cffi0/test_ownlib.py +++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py @@ -352,6 +352,8 @@ def test_modify_struct_value(self): if self.module is None: py.test.skip("fix the auto-generation of the tiny test lib") + if self.Backend is CTypesBackend: + py.test.skip("fails with the ctypes backend on some architectures") ffi = FFI(backend=self.Backend()) ffi.cdef(""" typedef struct { diff --git a/extra_tests/cffi_tests/cffi0/test_parsing.py b/extra_tests/cffi_tests/cffi0/test_parsing.py --- a/extra_tests/cffi_tests/cffi0/test_parsing.py +++ b/extra_tests/cffi_tests/cffi0/test_parsing.py @@ -467,3 +467,40 @@ e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}') assert str(e.value) == (':1: unexpected : ' 'this construct is valid C but not valid in cdef()') + +def test_unsigned_int_suffix_for_constant(): + ffi = FFI() + ffi.cdef("""enum e { + bin_0=0b10, + bin_1=0b10u, + bin_2=0b10U, + bin_3=0b10l, + bin_4=0b10L, + bin_5=0b10ll, + bin_6=0b10LL, + oct_0=010, + oct_1=010u, + oct_2=010U, + oct_3=010l, + oct_4=010L, + oct_5=010ll, + oct_6=010LL, + dec_0=10, + dec_1=10u, + dec_2=10U, + dec_3=10l, + dec_4=10L, + dec_5=10ll, + dec_6=10LL, + hex_0=0x10, + hex_1=0x10u, + hex_2=0x10U, + hex_3=0x10l, + hex_4=0x10L, + hex_5=0x10ll, + hex_6=0x10LL,};""") + needs_dlopen_none() + C = ffi.dlopen(None) + for base, expected_result in (('bin', 2), ('oct', 8), ('dec', 10), ('hex', 16)): + for index in range(7): + assert getattr(C, '{base}_{index}'.format(base=base, index=index)) == expected_result diff --git a/extra_tests/cffi_tests/cffi0/test_verify.py b/extra_tests/cffi_tests/cffi0/test_verify.py --- a/extra_tests/cffi_tests/cffi0/test_verify.py +++ b/extra_tests/cffi_tests/cffi0/test_verify.py @@ -1,5 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, re +import pytest import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model, FFIError from extra_tests.cffi_tests.support import * @@ -21,7 +22,8 @@ extra_compile_args.append('-Qunused-arguments') else: # assume a standard gcc - extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] + extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion', + '-Wno-unused-parameter'] class FFI(FFI): def verify(self, *args, **kwds): @@ -590,7 +592,8 @@ assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int') s = ffi.new("struct foo_s *") assert ffi.sizeof(s.a) == 17 * ffi.sizeof('int') - py.test.raises(IndexError, 's.a[17]') + with pytest.raises(IndexError): + s.a[17] def test_struct_array_c99_1(): if sys.platform == 'win32': @@ -648,7 +651,8 @@ ffi.verify("struct foo_s { int a:2, b:3; };") s = ffi.new("struct foo_s *") s.b = 3 - py.test.raises(OverflowError, "s.b = 4") + with pytest.raises(OverflowError): + s.b = 4 assert s.b == 3 def test_struct_with_bitfield_enum(): @@ -1464,8 +1468,10 @@ p = ffi.new("struct foo_s *") p.x = 1 assert p.x is True - py.test.raises(OverflowError, "p.x = -1") - py.test.raises(TypeError, "p.x = 0.0") + with pytest.raises(OverflowError): + p.x = -1 + with pytest.raises(TypeError): + p.x = 0.0 assert lib.foop(1) is False assert lib.foop(True) is False assert lib.foop(0) is True @@ -1533,7 +1539,8 @@ } """ % (type, type)) p = ffi.new("struct foo_s *") - py.test.raises(TypeError, "p.x = 0.0") + with pytest.raises(TypeError): + p.x = 0.0 assert lib.foo(42) == 0 assert lib.foo(0) == 1 py.test.raises(TypeError, lib.foo, 0.0) @@ -2099,6 +2106,11 @@ raise errors[0][1] def test_errno_working_even_with_pypys_jit(): + # NOTE: on some platforms, to work correctly, this test needs to be + # compiled with -pthread. Otherwise, the accesses to errno done from f() + # are compiled by assuming this small library won't be used from multiple + # threads, which is wrong. If you see failures _and_ if you pass your + # own CFLAGS environment variable, please make sure "-pthread" is in it. ffi = FFI() ffi.cdef("int f(int);") lib = ffi.verify(""" diff --git a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py --- a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py +++ b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py @@ -1,5 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py, sys +import pytest import _cffi_backend as _cffi1_backend @@ -86,9 +87,12 @@ def test_ffi_no_attr(): ffi = _cffi1_backend.FFI() - py.test.raises(AttributeError, "ffi.no_such_name") - py.test.raises(AttributeError, "ffi.no_such_name = 42") - py.test.raises(AttributeError, "del ffi.no_such_name") + with pytest.raises(AttributeError): + ffi.no_such_name + with pytest.raises(AttributeError): + ffi.no_such_name = 42 + with pytest.raises(AttributeError): + del ffi.no_such_name def test_ffi_string(): ffi = _cffi1_backend.FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py --- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py +++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py @@ -1,5 +1,6 @@ # Generated by pypy/tool/import_cffi.py import py +import pytest import platform, imp import sys, os, ctypes import cffi @@ -187,10 +188,14 @@ p[9] = 43 assert p[0] == 42 assert p[9] == 43 - py.test.raises(IndexError, "p[10]") - py.test.raises(IndexError, "p[10] = 44") - py.test.raises(IndexError, "p[-1]") - py.test.raises(IndexError, "p[-1] = 44") + with pytest.raises(IndexError): + p[10] + with pytest.raises(IndexError): + p[10] = 44 + with pytest.raises(IndexError): + p[-1] + with pytest.raises(IndexError): + p[-1] = 44 def test_new_array_args(self): # this tries to be closer to C: where we say "int x[5] = {10, 20, ..}" @@ -213,18 +218,21 @@ def test_new_array_varsize(self): p = ffi.new("int[]", 10) # a single integer is the length assert p[9] == 0 - py.test.raises(IndexError, "p[10]") + with pytest.raises(IndexError): + p[10] # py.test.raises(TypeError, ffi.new, "int[]") # p = ffi.new("int[]", [-6, -7]) # a list is all the items, like C assert p[0] == -6 assert p[1] == -7 - py.test.raises(IndexError, "p[2]") + with pytest.raises(IndexError): + p[2] assert repr(p) == "" % (2*SIZE_OF_INT) # p = ffi.new("int[]", 0) - py.test.raises(IndexError, "p[0]") + with pytest.raises(IndexError): + p[0] py.test.raises(ValueError, ffi.new, "int[]", -1) assert repr(p) == "" @@ -325,7 +333,8 @@ p[2][3] = 33 assert p[0][0] == 10 assert p[2][3] == 33 - py.test.raises(IndexError, "p[1][-1]") + with pytest.raises(IndexError): + p[1][-1] def test_constructor_array_of_array(self): p = ffi.new("int[3][2]", [[10, 11], [12, 13], [14, 15]]) @@ -446,7 +455,8 @@ n = ffi.new("int*", 99) p = ffi.new("int*[]", [n]) assert p[0][0] == 99 - py.test.raises(TypeError, "p[0] = None") + with pytest.raises(TypeError): + p[0] = None p[0] = ffi.NULL assert p[0] == ffi.NULL @@ -479,13 +489,15 @@ assert s.a == s.b == s.c == 0 s.b = -23 assert s.b == -23 - py.test.raises(OverflowError, "s.b = 32768") + with pytest.raises(OverflowError): + s.b = 32768 # s = ffi.new("struct simple*", [-2, -3]) assert s.a == -2 assert s.b == -3 assert s.c == 0 - py.test.raises((AttributeError, TypeError), "del s.a") + with pytest.raises((AttributeError, TypeError)): + del s.a assert repr(s) == "" % ( SIZE_OF_INT + 2 * SIZE_OF_SHORT) # @@ -503,8 +515,10 @@ assert s[0].a == s[0].b == s[0].c == 0 s[0].b = -23 assert s[0].b == s.b == -23 - py.test.raises(OverflowError, "s[0].b = -32769") - py.test.raises(IndexError, "s[1]") + with pytest.raises(OverflowError): + s[0].b = -32769 + with pytest.raises(IndexError): + s[1] def test_struct_opaque(self): py.test.raises(ffi.error, ffi.new, "struct baz*") @@ -556,11 +570,13 @@ u.b = -23 assert u.b == -23 assert u.a != 0 - py.test.raises(OverflowError, "u.b = 32768") + with pytest.raises(OverflowError): + u.b = 32768 # u = ffi.new("union simple_u*", [-2]) assert u.a == -2 - py.test.raises((AttributeError, TypeError), "del u.a") + with pytest.raises((AttributeError, TypeError)): + del u.a assert repr(u) == "" % ( SIZE_OF_INT,) @@ -626,7 +642,8 @@ p[3] = b'\x00' assert ffi.string(p) == b"hel" assert ffi.string(p, 2) == b"he" - py.test.raises(IndexError, "p[7] = b'X'") + with pytest.raises(IndexError): + p[7] = b'X' # a = ffi.new("char[]", b"hello\x00world") assert len(a) == 12 @@ -649,7 +666,8 @@ p[3] = u+'\x00' assert ffi.string(p) == u+"hel" assert ffi.string(p, 123) == u+"hel" - py.test.raises(IndexError, "p[7] = u+'X'") + with pytest.raises(IndexError): + p[7] = u+'X' # a = ffi.new("wchar_t[]", u+"hello\x00world") assert len(a) == 12 @@ -665,7 +683,8 @@ s = ffi.new("struct string*", [t]) assert type(s.name) not in (bytes, str, unicode) assert ffi.string(s.name) == b"testing" - py.test.raises(TypeError, "s.name = None") + with pytest.raises(TypeError): + s.name = None s.name = ffi.NULL assert s.name == ffi.NULL @@ -686,17 +705,20 @@ a = ffi.new("int[]", [10, 11, 12]) p = ffi.new("void **", a) vp = p[0] - py.test.raises(TypeError, "vp[0]") + with pytest.raises(TypeError): + vp[0] py.test.raises(TypeError, ffi.new, "short **", a) # s = ffi.new("struct voidp *") s.p = a # works s.q = a # works - py.test.raises(TypeError, "s.r = a") # fails + with pytest.raises(TypeError): + s.r = a # fails b = ffi.cast("int *", a) s.p = b # works s.q = b # works - py.test.raises(TypeError, "s.r = b") # fails + with pytest.raises(TypeError): + s.r = b # fails def test_functionptr_simple(self): py.test.raises(TypeError, ffi.callback, "int(*)(int)", 0) @@ -714,7 +736,8 @@ q = ffi.new("int(**)(int)", p) assert repr(q) == "" % ( SIZE_OF_PTR) - py.test.raises(TypeError, "q(43)") + with pytest.raises(TypeError): + q(43) res = q[0](43) assert res == 44 q = ffi.cast("int(*)(int)", p) @@ -923,10 +946,14 @@ assert s.e in (4294967295, -1) # two choices assert s[0].e in (4294967295, -1) s.e = s.e - py.test.raises(TypeError, "s.e = 'B3'") - py.test.raises(TypeError, "s.e = '2'") - py.test.raises(TypeError, "s.e = '#2'") - py.test.raises(TypeError, "s.e = '#7'") + with pytest.raises(TypeError): + s.e = 'B3' + with pytest.raises(TypeError): + s.e = '2' + with pytest.raises(TypeError): + s.e = '#2' + with pytest.raises(TypeError): + s.e = '#7' def test_enum_non_contiguous(self): # enum noncont { A4, B4=42, C4 }; @@ -948,11 +975,14 @@ def test_array_of_struct(self): s = ffi.new("struct ab[1]") - py.test.raises(AttributeError, 's.b') - py.test.raises(AttributeError, 's.b = 412') + with pytest.raises(AttributeError): + s.b + with pytest.raises(AttributeError): + s.b = 412 s[0].b = 412 assert s[0].b == 412 - py.test.raises(IndexError, 's[1]') + with pytest.raises(IndexError): + s[1] def test_pointer_to_array(self): p = ffi.new("int(**)[5]") @@ -1001,17 +1031,23 @@ assert ffi.sizeof("struct bitfield") == 8 s = ffi.new("struct bitfield *") s.a = 511 - py.test.raises(OverflowError, "s.a = 512") - py.test.raises(OverflowError, "s[0].a = 512") + with pytest.raises(OverflowError): + s.a = 512 + with pytest.raises(OverflowError): + s[0].a = 512 assert s.a == 511 s.a = -512 - py.test.raises(OverflowError, "s.a = -513") - py.test.raises(OverflowError, "s[0].a = -513") + with pytest.raises(OverflowError): + s.a = -513 + with pytest.raises(OverflowError): + s[0].a = -513 assert s.a == -512 s.c = 3 assert s.c == 3 - py.test.raises(OverflowError, "s.c = 4") - py.test.raises(OverflowError, "s[0].c = 4") + with pytest.raises(OverflowError): + s.c = 4 + with pytest.raises(OverflowError): + s[0].c = 4 s.c = -4 assert s.c == -4 @@ -1236,7 +1272,8 @@ p = ffi.new("struct foo_s *", 10) # a single integer is the length assert p.len == 0 assert p.data[9] == 0 - py.test.raises(IndexError, "p.data[10]") + with pytest.raises(IndexError): + p.data[10] def test_ffi_typeof_getcname(self): assert ffi.getctype("int") == "int" @@ -1753,7 +1790,8 @@ assert MYFOO == 42 assert myfunc(43) == 44 assert myvar == -5 # but can't be changed, so not very useful - py.test.raises(ImportError, "from _test_import_from_lib.lib import bar") + with pytest.raises(ImportError): + from _test_import_from_lib.lib import bar d = {} exec("from _test_import_from_lib.lib import *", d) assert (set(key for key in d if not key.startswith('_')) == diff --git a/extra_tests/cffi_tests/cffi1/test_pkgconfig.py b/extra_tests/cffi_tests/cffi1/test_pkgconfig.py new file mode 100644 --- /dev/null +++ b/extra_tests/cffi_tests/cffi1/test_pkgconfig.py @@ -0,0 +1,95 @@ +# Generated by pypy/tool/import_cffi.py +import sys +import subprocess +import py +import cffi.pkgconfig as pkgconfig +from cffi import PkgConfigError + + +def mock_call(libname, flag): + assert libname=="foobarbaz" + flags = { + "--cflags": "-I/usr/include/python3.6m -DABCD -DCFFI_TEST=1 -O42\n", + "--libs": "-L/usr/lib64 -lpython3.6 -shared\n", + } + return flags[flag] + + +def test_merge_flags(): + d1 = {"ham": [1, 2, 3], "spam" : ["a", "b", "c"], "foo" : []} + d2 = {"spam" : ["spam", "spam", "spam"], "bar" : ["b", "a", "z"]} + + pkgconfig.merge_flags(d1, d2) + assert d1 == { + "ham": [1, 2, 3], + "spam" : ["a", "b", "c", "spam", "spam", "spam"], + "bar" : ["b", "a", "z"], + "foo" : []} + + +def test_pkgconfig(): + assert pkgconfig.flags_from_pkgconfig([]) == {} + + saved = pkgconfig.call + try: + pkgconfig.call = mock_call + flags = pkgconfig.flags_from_pkgconfig(["foobarbaz"]) + finally: + pkgconfig.call = saved + assert flags == { + 'include_dirs': ['/usr/include/python3.6m'], + 'library_dirs': ['/usr/lib64'], + 'libraries': ['python3.6'], + 'define_macros': [('ABCD', None), ('CFFI_TEST', '1')], + 'extra_compile_args': ['-O42'], + 'extra_link_args': ['-shared'] + } + +class mock_subprocess: + PIPE = Ellipsis + class Popen: + def __init__(self, cmd, stdout, stderr): + if mock_subprocess.RESULT is None: + raise OSError("oops can't run") + assert cmd == ['pkg-config', '--print-errors', '--cflags', 'libfoo'] + def communicate(self): + bout, berr, rc = mock_subprocess.RESULT + self.returncode = rc + return bout, berr + +def test_call(): + saved = pkgconfig.subprocess + try: + pkgconfig.subprocess = mock_subprocess + + mock_subprocess.RESULT = None + e = py.test.raises(PkgConfigError, pkgconfig.call, "libfoo", "--cflags") + assert str(e.value) == "cannot run pkg-config: oops can't run" + + mock_subprocess.RESULT = b"", "Foo error!\n", 1 + e = py.test.raises(PkgConfigError, pkgconfig.call, "libfoo", "--cflags") + assert str(e.value) == "Foo error!" + + mock_subprocess.RESULT = b"abc\\def\n", "", 0 + e = py.test.raises(PkgConfigError, pkgconfig.call, "libfoo", "--cflags") + assert str(e.value).startswith("pkg-config --cflags libfoo returned an " + "unsupported backslash-escaped output:") + + mock_subprocess.RESULT = b"abc def\n", "", 0 + result = pkgconfig.call("libfoo", "--cflags") + assert result == "abc def\n" + + mock_subprocess.RESULT = b"abc def\n", "", 0 + result = pkgconfig.call("libfoo", "--cflags") + assert result == "abc def\n" + + if sys.version_info >= (3,): + mock_subprocess.RESULT = b"\xff\n", "", 0 + e = py.test.raises(PkgConfigError, pkgconfig.call, + "libfoo", "--cflags", encoding="utf-8") + assert str(e.value) == ( + "pkg-config --cflags libfoo returned bytes that cannot be " + "decoded with encoding 'utf-8':\nb'\\xff\\n'") + + finally: + pkgconfig.subprocess = saved diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -1,6 +1,7 @@ # Generated by pypy/tool/import_cffi.py import sys, os, py +import pytest from cffi import FFI, VerificationError, FFIError, CDefError from cffi import recompiler from extra_tests.cffi_tests.udir import udir @@ -189,20 +190,26 @@ assert lib.a == -2 lib.a = -2147483648 assert lib.a == -2147483648 - py.test.raises(OverflowError, "lib.a = 2147483648") - py.test.raises(OverflowError, "lib.a = -2147483649") + with pytest.raises(OverflowError): + lib.a = 2147483648 + with pytest.raises(OverflowError): + lib.a = -2147483649 lib.b = 525 # try with the first access being in setattr, too assert lib.b == 525 - py.test.raises(AttributeError, "del lib.a") - py.test.raises(AttributeError, "del lib.c") - py.test.raises(AttributeError, "del lib.foobarbaz") + with pytest.raises(AttributeError): + del lib.a + with pytest.raises(AttributeError): + del lib.c + with pytest.raises(AttributeError): + del lib.foobarbaz def test_macro(): ffi = FFI() ffi.cdef("#define FOOBAR ...") lib = verify(ffi, 'test_macro', "#define FOOBAR (-6912)") assert lib.FOOBAR == -6912 - py.test.raises(AttributeError, "lib.FOOBAR = 2") + with pytest.raises(AttributeError): + lib.FOOBAR = 2 def test_macro_check_value(): # the value '-0x80000000' in C sources does not have a clear meaning @@ -248,7 +255,8 @@ ffi.cdef("static const int FOOBAR;") lib = verify(ffi, 'test_constant', "#define FOOBAR (-6912)") assert lib.FOOBAR == -6912 - py.test.raises(AttributeError, "lib.FOOBAR = 2") + with pytest.raises(AttributeError): + lib.FOOBAR = 2 def test_check_value_of_static_const(): ffi = FFI() @@ -264,7 +272,8 @@ ffi.cdef("static const double FOOBAR;") lib = verify(ffi, 'test_constant_nonint', "#define FOOBAR (-6912.5)") assert lib.FOOBAR == -6912.5 - py.test.raises(AttributeError, "lib.FOOBAR = 2") + with pytest.raises(AttributeError): + lib.FOOBAR = 2 def test_constant_ptr(): ffi = FFI() @@ -316,8 +325,10 @@ p = ffi.new("struct foo_s *", {'a': -32768, 'b': -2147483648}) assert p.a == -32768 assert p.b == -2147483648 - py.test.raises(OverflowError, "p.a -= 1") - py.test.raises(OverflowError, "p.b -= 1") + with pytest.raises(OverflowError): + p.a -= 1 + with pytest.raises(OverflowError): + p.b -= 1 q = ffi.new("struct bar_s *", {'f': p}) assert q.f == p # @@ -388,8 +399,10 @@ assert ffi.sizeof("struct foo_s") == (42 + 11) * 4 p = ffi.new("struct foo_s *") assert p.a[41] == p.b[10] == 0 - py.test.raises(IndexError, "p.a[42]") - py.test.raises(IndexError, "p.b[11]") + with pytest.raises(IndexError): + p.a[42] + with pytest.raises(IndexError): + p.b[11] def test_dotdotdot_global_array(): ffi = FFI() @@ -399,8 +412,10 @@ assert ffi.sizeof(lib.aa) == 41 * 4 assert ffi.sizeof(lib.bb) == 12 * 4 assert lib.aa[40] == lib.bb[11] == 0 - py.test.raises(IndexError, "lib.aa[41]") - py.test.raises(IndexError, "lib.bb[12]") + with pytest.raises(IndexError): + lib.aa[41] + with pytest.raises(IndexError): + lib.bb[12] def test_misdeclared_field_1(): ffi = FFI() @@ -1021,8 +1036,10 @@ assert ffi.typeof(s.a) == ffi.typeof("int[5][8]") assert ffi.sizeof(s.a) == 40 * ffi.sizeof('int') assert s.a[4][7] == 0 - py.test.raises(IndexError, 's.a[4][8]') - py.test.raises(IndexError, 's.a[5][0]') + with pytest.raises(IndexError): + s.a[4][8] + with pytest.raises(IndexError): + s.a[5][0] assert ffi.typeof(s.a) == ffi.typeof("int[5][8]") assert ffi.typeof(s.a[0]) == ffi.typeof("int[8]") @@ -1035,7 +1052,8 @@ s = ffi.new("struct foo_s *") assert ffi.typeof(s.a) == ffi.typeof("int[][7]") assert s.a[4][6] == 0 - py.test.raises(IndexError, 's.a[4][7]') + with pytest.raises(IndexError): + s.a[4][7] assert ffi.typeof(s.a[0]) == ffi.typeof("int[7]") def test_global_var_array_2(): @@ -1044,8 +1062,10 @@ lib = verify(ffi, 'test_global_var_array_2', 'int a[10][8];') lib.a[9][7] = 123456 assert lib.a[9][7] == 123456 - py.test.raises(IndexError, 'lib.a[0][8]') - py.test.raises(IndexError, 'lib.a[10][0]') + with pytest.raises(IndexError): + lib.a[0][8] + with pytest.raises(IndexError): + lib.a[10][0] assert ffi.typeof(lib.a) == ffi.typeof("int[10][8]") assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]") @@ -1055,7 +1075,8 @@ lib = verify(ffi, 'test_global_var_array_3', 'int a[10][8];') lib.a[9][7] = 123456 assert lib.a[9][7] == 123456 - py.test.raises(IndexError, 'lib.a[0][8]') + with pytest.raises(IndexError): + lib.a[0][8] assert ffi.typeof(lib.a) == ffi.typeof("int(*)[8]") assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]") @@ -1065,8 +1086,10 @@ lib = verify(ffi, 'test_global_var_array_4', 'int a[10][8];') lib.a[9][7] = 123456 assert lib.a[9][7] == 123456 - py.test.raises(IndexError, 'lib.a[0][8]') - py.test.raises(IndexError, 'lib.a[10][8]') + with pytest.raises(IndexError): + lib.a[0][8] + with pytest.raises(IndexError): + lib.a[10][8] assert ffi.typeof(lib.a) == ffi.typeof("int[10][8]") assert ffi.typeof(lib.a[0]) == ffi.typeof("int[8]") @@ -1339,7 +1362,8 @@ #define aaa 42 """) assert lib.aaa == 42 - py.test.raises(AttributeError, "lib.aaa = 43") + with pytest.raises(AttributeError): + lib.aaa = 43 def test_win32_calling_convention_0(): ffi = FFI() diff --git a/extra_tests/cffi_tests/cffi1/test_verify1.py b/extra_tests/cffi_tests/cffi1/test_verify1.py --- a/extra_tests/cffi_tests/cffi1/test_verify1.py +++ b/extra_tests/cffi_tests/cffi1/test_verify1.py @@ -1,5 +1,6 @@ # Generated by pypy/tool/import_cffi.py import os, sys, math, py +import pytest from cffi import FFI, FFIError, VerificationError, VerificationMissing, model from cffi import CDefError from cffi import recompiler @@ -23,7 +24,8 @@ extra_compile_args.append('-Qunused-arguments') else: # assume a standard gcc - extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion'] + extra_compile_args = ['-Werror', '-Wall', '-Wextra', '-Wconversion', + '-Wno-unused-parameter'] class FFI(FFI): error = _cffi_backend.FFI.error @@ -572,7 +574,8 @@ assert ffi.sizeof('struct foo_s') == 19 * ffi.sizeof('int') s = ffi.new("struct foo_s *") assert ffi.sizeof(s.a) == 17 * ffi.sizeof('int') - py.test.raises(IndexError, 's.a[17]') + with pytest.raises(IndexError): + s.a[17] def test_struct_array_c99_1(): if sys.platform == 'win32': @@ -630,7 +633,8 @@ ffi.verify("struct foo_s { int a:2, b:3; };") s = ffi.new("struct foo_s *") s.b = 3 - py.test.raises(OverflowError, "s.b = 4") + with pytest.raises(OverflowError): + s.b = 4 assert s.b == 3 def test_struct_with_bitfield_enum(): @@ -1434,8 +1438,10 @@ p = ffi.new("struct foo_s *") p.x = 1 assert p.x is True - py.test.raises(OverflowError, "p.x = -1") - py.test.raises(TypeError, "p.x = 0.0") + with pytest.raises(OverflowError): + p.x = -1 + with pytest.raises(TypeError): + p.x = 0.0 assert lib.foop(1) is False assert lib.foop(True) is False assert lib.foop(0) is True @@ -1503,7 +1509,8 @@ } """ % (type, type)) p = ffi.new("struct foo_s *") - py.test.raises(TypeError, "p.x = 0.0") + with pytest.raises(TypeError): + p.x = 0.0 assert lib.foo(42) == 0 assert lib.foo(0) == 1 py.test.raises(TypeError, lib.foo, 0.0) @@ -2194,7 +2201,8 @@ ffi = FFI() ffi.cdef("#define FOO 123") lib = ffi.verify("#define FOO 124") # used to complain - e = py.test.raises(ffi.error, "lib.FOO") + with pytest.raises(ffi.error) as e: + lib.FOO assert str(e.value) == ("the C compiler says 'FOO' is equal to 124 (0x7c)," " but the cdef disagrees") diff --git a/extra_tests/cffi_tests/embedding/test_basic.py b/extra_tests/cffi_tests/embedding/test_basic.py --- a/extra_tests/cffi_tests/embedding/test_basic.py +++ b/extra_tests/cffi_tests/embedding/test_basic.py @@ -173,7 +173,8 @@ result = popen.stdout.read() err = popen.wait() if err: - raise OSError("%r failed with exit code %r" % (name, err)) + raise OSError("%r failed with exit code %r" % ( + os.path.join(path, executable_name), err)) return result diff --git a/extra_tests/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c --- a/extra_tests/cffi_tests/embedding/thread3-test.c +++ b/extra_tests/cffi_tests/embedding/thread3-test.c @@ -52,5 +52,6 @@ assert(status == 0); } printf("done\n"); + fflush(stdout); /* this is occasionally needed on Windows */ return 0; } diff --git a/extra_tests/ctypes_tests/test_structures.py b/extra_tests/ctypes_tests/test_structures.py --- a/extra_tests/ctypes_tests/test_structures.py +++ b/extra_tests/ctypes_tests/test_structures.py @@ -124,12 +124,15 @@ ms.n = 0xff00 return repr(ba[:]) + nstruct = dostruct(Native) if sys.byteorder == 'little': - assert dostruct(Native) == dostruct(Little) - assert dostruct(Native) != dostruct(Big) + assert nstruct == dostruct(Little) + assert nstruct != dostruct(Big) + assert Big._fields_[0][1] is not i else: - assert dostruct(Native) == dostruct(Big) - assert dostruct(Native) != dostruct(Little) + assert nstruct == dostruct(Big) + assert nstruct != dostruct(Little) + assert Little._fields_[0][1] is not i def test_from_buffer_copy(): from array import array @@ -190,3 +193,20 @@ assert sizeof(s) == 3 * sizeof(c_int) assert s.a == 4 # 256 + 4 assert s.b == -123 + +def test_memoryview(): + class S(Structure): + _fields_ = [('a', c_int16), + ('b', c_int16), + ] + + S3 = S * 3 + c_array = (2 * S3)( + S3(S(a=0, b=1), S(a=2, b=3), S(a=4, b=5)), + S3(S(a=6, b=7), S(a=8, b=9), S(a=10, b=11)), + ) + + mv = memoryview(c_array) + assert mv.format == 'T{= (1, 1, 0): default |= ssl.OP_NO_COMPRESSION + default |= ssl.OP_ENABLE_MIDDLEBOX_COMPAT self.assertEqual(default, ctx.options) ctx.options |= ssl.OP_NO_TLSv1 self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) diff --git a/lib-python/2.7/test/test_sys.py b/lib-python/2.7/test/test_sys.py --- a/lib-python/2.7/test/test_sys.py +++ b/lib-python/2.7/test/test_sys.py @@ -216,6 +216,11 @@ raise ValueError() except ValueError, e: pass + except MemoryError: + # Documentation for setrecursionlimit says: "The highest possible + # limit is platform-dependent. ... a too-high limit can lead to a + # crash" so we allow MemoryError here + pass finally: sys.setrecursionlimit(oldlimit) diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py --- a/lib-python/2.7/test/test_unicode.py +++ b/lib-python/2.7/test/test_unicode.py @@ -1652,10 +1652,10 @@ # when a string allocation fails with a MemoryError. # This used to crash the interpreter, # or leak references when the number was smaller. - charwidth = 4 if sys.maxunicode >= 0x10000 else 2 + charwidth = 2 # pypy: the char \u0123 is stored in two utf-8 bytes # Note: sys.maxsize is half of the actual max allocation because of # the signedness of Py_ssize_t. - alloc = lambda: u"a" * (sys.maxsize // charwidth * 2) + alloc = lambda: u"\u0123" * (sys.maxsize // charwidth * 2) self.assertRaises(MemoryError, alloc) self.assertRaises(MemoryError, alloc) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -176,7 +176,7 @@ RegrTest('test_copy_reg.py', core=True), RegrTest('test_cpickle.py', core=True), RegrTest('test_cprofile.py'), - RegrTest('test_crypt.py', usemodules='crypt'), + RegrTest('test_crypt.py'), RegrTest('test_csv.py', usemodules='_csv'), RegrTest('test_ctypes.py', usemodules="_rawffi thread cpyext"), RegrTest('test_curses.py'), @@ -401,7 +401,7 @@ RegrTest('test_source_encoding.py'), RegrTest('test_spwd.py'), RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), - RegrTest('test_ssl.py', usemodules='_ssl _socket select'), + RegrTest('test_ssl.py', usemodules='_socket select'), RegrTest('test_startfile.py'), RegrTest('test_stat.py'), RegrTest('test_str.py', core=True), diff --git a/lib_pypy/_cffi_ssl/LICENSE b/lib_pypy/_cffi_ssl/LICENSE new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/LICENSE @@ -0,0 +1,26 @@ + +Except when otherwise stated (look for LICENSE files in directories or +information at the beginning of each file) all software and +documentation is licensed as follows: + + The MIT License + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/README.md @@ -0,0 +1,35 @@ +# PyPy's SSL module + +All of the CFFI code is copied from cryptography, wich patches contributed +back to cryptography. PyPy vendors it's own copy of the cffi backend thus +it renames the compiled shared object to _pypy_openssl.so (which means +that cryptography can ship their own cffi backend) + +NOTE: currently, we have the following changes: + +* ``_cffi_src/openssl/callbacks.py`` to not rely on the CPython C API + (this change is now backported) + +* ``_cffi_src/utils.py`` for issue #2575 (29c9a89359e4) + +* ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1) + +* ``_cffi_src/openssl/pypy_win32_extra.py`` for Win32-only functionality like ssl.enum_certificates() + + +# Tests? + +Currently this module is tested using CPython's standard library test suite. + +# Install it into PyPy's source tree + +Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command:: + + $ cp -r /src/_cffi_src/* . + +NOTE: you need to keep our version of ``_cffi_src/openssl/callbacks.py`` +for now! + +# Crpytography version + +Copied over release version `1.7.2` diff --git a/lib_pypy/_cffi_ssl/__init__.py b/lib_pypy/_cffi_ssl/__init__.py new file mode 100644 diff --git a/lib_pypy/_cffi_ssl/_cffi_src/__init__.py b/lib_pypy/_cffi_ssl/_cffi_src/__init__.py new file mode 100644 diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/build_commoncrypto.py @@ -0,0 +1,33 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from _cffi_src.utils import build_ffi_for_binding + + +ffi = build_ffi_for_binding( + module_name="_commoncrypto", + module_prefix="_cffi_src.commoncrypto.", + modules=[ + "cf", + "common_digest", + "common_hmac", + "common_key_derivation", + "common_cryptor", + "common_symmetric_key_wrap", + "seccertificate", + "secimport", + "secitem", + "seckey", + "seckeychain", + "secpolicy", + "sectransform", + "sectrust", + "secure_transport", + ], + extra_link_args=[ + "-framework", "Security", "-framework", "CoreFoundation" + ], +) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/build_constant_time.py @@ -0,0 +1,27 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import os + +from _cffi_src.utils import build_ffi, compiler_type, extra_link_args + + +with open(os.path.join( + os.path.dirname(__file__), "hazmat_src/constant_time.h" +)) as f: + types = f.read() + +with open(os.path.join( + os.path.dirname(__file__), "hazmat_src/constant_time.c" +)) as f: + functions = f.read() + +ffi = build_ffi( + module_name="_constant_time", + cdef_source=types, + verify_source=functions, + extra_link_args=extra_link_args(compiler_type()), +) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/_cffi_src/build_openssl.py From pypy.commits at gmail.com Wed Jun 5 07:11:02 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 05 Jun 2019 04:11:02 -0700 (PDT) Subject: [pypy-commit] pypy arm64: fixes Message-ID: <5cf7a346.1c69fb81.ab72e.98b7@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96752:b5abcfa99f96 Date: 2019-06-05 11:10 +0000 http://bitbucket.org/pypy/pypy/changeset/b5abcfa99f96/ Log: fixes diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -209,6 +209,22 @@ index = op.getarg(0).getint() self.load_from_gc_table(res_loc.value, index) + def emit_op_load_effective_address(self, op, arglocs): + self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[3].value, + arglocs[2].value) + + # result = base_loc + (scaled_loc << scale) + static_offset + def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0): + assert scaled_loc.is_core_reg() + assert base_loc.is_core_reg() + if scale > 0: + self.mc.LSL_ri(r.ip0.value, scaled_loc.value, scale) + scaled_loc = r.ip0 + else: + scaled_loc = scaled_loc + self.mc.ADD_rr(result.value, base_loc.value, scaled_loc.value) + self.mc.ADD_ri(result.value, result.value, static_offset) + def emit_op_debug_merge_point(self, op, arglocs): pass diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -180,9 +180,8 @@ operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, allgcrefs) # compute longevity of variables - longevity, last_real_usage = compute_vars_longevity(inputargs, operations) + longevity = compute_vars_longevity(inputargs, operations) self.longevity = longevity - self.last_real_usage = last_real_usage fm = self.frame_manager asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) @@ -602,7 +601,7 @@ position = self.rm.position for arg in inputargs: assert not isinstance(arg, Const) - if self.last_real_usage.get(arg, -1) <= position: + if self.longevity[arg].is_last_real_use_before(position): self.force_spill_var(arg) # @@ -724,6 +723,13 @@ resloc = self.force_allocate_reg(op) return [resloc] + def prepare_op_load_effective_address(self, op): + args = op.getarglist() + arg0 = self.make_sure_var_in_reg(args[0], args) + arg1 = self.make_sure_var_in_reg(args[1], args) + res = self.force_allocate_reg(op) + return [arg0, arg1, args[2], args[3], res] + def prepare_op_jump(self, op): assert self.jump_target_descr is None descr = op.getdescr() diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1057,6 +1057,7 @@ 'LOAD_FROM_GC_TABLE/1/r', # only emitted by rewrite.py # 'LOAD_EFFECTIVE_ADDRESS/4/i', # only emitted by rewrite.py, shortcut for x86 + # res = arg0 + (arg1 << arg3) + arg2 # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- From pypy.commits at gmail.com Wed Jun 5 07:18:53 2019 From: pypy.commits at gmail.com (fijal) Date: Wed, 05 Jun 2019 04:18:53 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: fixes from Aarch64 Message-ID: <5cf7a51d.1c69fb81.7b19d.7f47@mx.google.com> Author: fijal Branch: copystrcontents-in-rewrite Changeset: r96753:2fb764a56483 Date: 2019-06-05 13:18 +0200 http://bitbucket.org/pypy/pypy/changeset/2fb764a56483/ Log: fixes from Aarch64 diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -835,7 +835,8 @@ assert 0 def emit_op_load_effective_address(self, op, arglocs, regalloc, fcond): - self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[2], arglocs[3]) + self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[3].value, + arglocs[2].value) # result = base_loc + (scaled_loc << scale) + static_offset def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0): From pypy.commits at gmail.com Wed Jun 5 08:45:16 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 05:45:16 -0700 (PDT) Subject: [pypy-commit] pypy default: various improvements to the json decoder, particularly for big JSON files: Message-ID: <5cf7b95c.1c69fb81.1c6e1.bbbd@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r96754:cefcc54d57f7 Date: 2019-06-05 14:44 +0200 http://bitbucket.org/pypy/pypy/changeset/cefcc54d57f7/ Log: various improvements to the json decoder, particularly for big JSON files: - use maps for those objects that are instance-like (some heuristics to determine which are and which aren't) - maps use speculative parsing to try to parse the next-common key with strcmp - use a string cache to cache commonly reappearing strings, but only for instance fields where that is useful - use an int cache for small integers - use SIMD to find the end of strings efficiently still needs a bit of cleanup, but slowly getting there diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -1,11 +1,13 @@ import sys from rpython.rlib.rstring import StringBuilder -from rpython.rlib.objectmodel import specialize, always_inline, r_dict -from rpython.rlib import rfloat, rutf8 +from rpython.rlib.objectmodel import specialize, always_inline +from rpython.rlib import rfloat, runicode, jit, objectmodel, rutf8 from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import r_uint from pypy.interpreter.error import oefmt from pypy.interpreter import unicodehelper +from pypy.interpreter.baseobjspace import W_Root +from pypy.module._pypyjson import simd OVF_DIGITS = len(str(sys.maxint)) @@ -15,45 +17,85 @@ # precomputing negative powers of 10 is MUCH faster than using e.g. math.pow # at runtime NEG_POW_10 = [10.0**-i for i in range(16)] +del i + def neg_pow_10(x, exp): if exp >= len(NEG_POW_10): return 0.0 return x * NEG_POW_10[exp] -def slice_eq(a, b): - (ll_chars1, start1, length1, _) = a - (ll_chars2, start2, length2, _) = b - if length1 != length2: +def _compare_cache_entry(space, res, ll_chars, start, length): + if length != len(res): return False - j = start2 - for i in range(start1, start1 + length1): - if ll_chars1[i] != ll_chars2[j]: + index = start + for c in res: + x = ord(c) + if not ll_chars[index] == chr(x): return False - j += 1 + index += 1 return True -def slice_hash(a): - (ll_chars, start, length, h) = a - return h -TYPE_UNKNOWN = 0 -TYPE_STRING = 1 -class JSONDecoder(object): +class IntCache(object): + START = -10 + END = 256 + + def __init__(self, space): + self.space = space + self.cache = [self.space.newint(i) + for i in range(self.START, self.END)] + + def newint(self, intval): + if self.START <= intval < self.END: + return self.cache[intval - self.START] + return self.space.newint(intval) + + +class JSONDecoder(W_Root): + + LRU_SIZE = 16 + LRU_MASK = LRU_SIZE - 1 + + DEFAULT_SIZE_SCRATCH = 20 + + MIN_SIZE_FOR_STRING_CACHE = 1024 * 1024 + + def __init__(self, space, s): self.space = space + self.w_empty_string = space.newutf8("", 0) + self.s = s + # we put our string in a raw buffer so: # 1) we automatically get the '\0' sentinel at the end of the string, # which means that we never have to check for the "end of string" # 2) we can pass the buffer directly to strtod - self.ll_chars = rffi.str2charp(s) + self.ll_chars, self.flag = rffi.get_nonmovingbuffer_final_null(self.s) self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') self.pos = 0 - self.cache = r_dict(slice_eq, slice_hash, simple_hash_eq=True) + self.intcache = space.fromcache(IntCache) + + self.cache = {} + self.cache_wrapped = {} + + self.lru_cache = [0] * self.LRU_SIZE + self.lru_index = 0 + + self.startmap = self.space.fromcache(Terminator) + self.unclear_objects = [] + + self.scratch = [[None] * self.DEFAULT_SIZE_SCRATCH] # list of scratch space + def close(self): - rffi.free_charp(self.ll_chars) + rffi.free_nonmovingbuffer(self.s, self.ll_chars, self.flag) lltype.free(self.end_ptr, flavor='raw') + # clean up objects that are instances of now blocked maps + for w_obj in self.unclear_objects: + jsonmap = self._get_jsonmap_from_dict(w_obj) + if jsonmap.is_blocked(): + self._devolve_jsonmap_dict(w_obj) def getslice(self, start, end): assert start >= 0 @@ -61,18 +103,15 @@ return self.s[start:end] def skip_whitespace(self, i): + ll_chars = self.ll_chars while True: - ch = self.ll_chars[i] + ch = ll_chars[i] if is_whitespace(ch): - i+=1 + i += 1 else: break return i - @specialize.arg(1) - def _raise(self, msg, *args): - raise oefmt(self.space.w_ValueError, msg, *args) - def decode_any(self, i): i = self.skip_whitespace(i) ch = self.ll_chars[i] @@ -102,6 +141,11 @@ self._raise("No JSON object could be decoded: unexpected '%s' at char %d", ch, i) + + @specialize.arg(1) + def _raise(self, msg, *args): + raise oefmt(self.space.w_ValueError, msg, *args) + def decode_null(self, i): if (self.ll_chars[i] == 'u' and self.ll_chars[i+1] == 'l' and @@ -162,7 +206,7 @@ return self.decode_int_slow(start) self.pos = i - return self.space.newint(intval) + return self.intcache.newint(intval) def decode_float(self, i): from rpython.rlib import rdtoa @@ -214,6 +258,26 @@ ovf_maybe = (count >= OVF_DIGITS) return i, ovf_maybe, sign * intval + def _raise_control_char_in_string(self, ch, startindex, currindex): + if ch == '\0': + self._raise("Unterminated string starting at char %d", + startindex - 1) + else: + self._raise("Invalid control character at char %d", currindex-1) + + def _raise_object_error(self, ch, start, i): + if ch == '\0': + self._raise("Unterminated object starting at char %d", start) + else: + self._raise("Unexpected '%s' when decoding object (char %d)", + ch, i) + + def decode_surrogate_pair(self, i, highsurr): + """ uppon enter the following must hold: + chars[i] == "\\" and chars[i+1] == "u" + """ + # the possible ValueError is caught by the caller + def decode_array(self, i): w_list = self.space.newlist([]) start = i @@ -240,6 +304,13 @@ self._raise("Unexpected '%s' when decoding array (char %d)", ch, i-1) + def decode_any_context(self, i, context): + i = self.skip_whitespace(i) + ch = self.ll_chars[i] + if ch == '"': + return self.decode_string(i+1, context) + return self.decode_any(i) + def decode_object(self, i): start = i @@ -248,62 +319,124 @@ self.pos = i+1 return self.space.newdict() - d = self._create_empty_dict() + if self.scratch: + values_w = self.scratch.pop() + else: + values_w = [None] * self.DEFAULT_SIZE_SCRATCH + nextindex = 0 + currmap = self.startmap while True: # parse a key: value - w_name = self.decode_key(i) + currmap = self.decode_key(i, currmap) i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] if ch != ':': self._raise("No ':' found at char %d", i) i += 1 - i = self.skip_whitespace(i) - # - w_value = self.decode_any(i) - d[w_name] = w_value + + w_value = self.decode_any_context(i, currmap) + + if nextindex == len(values_w): # full + values_w = values_w + [None] * len(values_w) # double + values_w[nextindex] = w_value + nextindex += 1 i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] i += 1 if ch == '}': self.pos = i - return self._create_dict(d) + if currmap.is_blocked(): + currmap.instantiation_count += 1 + self.scratch.append(values_w) # can reuse next time + dict_w = self._switch_to_dict(currmap, values_w, nextindex) + return self._create_dict(dict_w) + self.scratch.append(values_w) # can reuse next time + values_w = values_w[:nextindex] + currmap.instantiation_count += 1 + w_res = self._create_dict_map(values_w, currmap) + if currmap.state != MapBase.USEFUL: + self.unclear_objects.append(w_res) + return w_res elif ch == ',': - pass - elif ch == '\0': - self._raise("Unterminated object starting at char %d", start) + i = self.skip_whitespace(i) + if currmap.is_blocked(): + currmap.instantiation_count += 1 + self.scratch.append(values_w) # can reuse next time + dict_w = self._switch_to_dict(currmap, values_w, nextindex) + return self.decode_object_dict(i, start, dict_w) else: - self._raise("Unexpected '%s' when decoding object (char %d)", - ch, i-1) + self._raise_object_error(ch, start, i - 1) - def decode_string(self, i): - start = i - bits = 0 + def _create_dict_map(self, values_w, jsonmap): + from pypy.objspace.std.jsondict import from_values_and_jsonmap + return from_values_and_jsonmap(self.space, values_w, jsonmap) + + def _devolve_jsonmap_dict(self, w_dict): + from pypy.objspace.std.jsondict import devolve_jsonmap_dict + devolve_jsonmap_dict(w_dict) + + def _get_jsonmap_from_dict(self, w_dict): + from pypy.objspace.std.jsondict import get_jsonmap_from_dict + return get_jsonmap_from_dict(w_dict) + + def _switch_to_dict(self, currmap, values_w, nextindex): + dict_w = self._create_empty_dict() + index = nextindex - 1 + while isinstance(currmap, JSONMap): + dict_w[currmap.w_key] = values_w[index] + index -= 1 + currmap = currmap.prev + assert len(dict_w) == nextindex + return dict_w + + def decode_object_dict(self, i, start, dict_w): while True: - # this loop is a fast path for strings which do not contain escape - # characters + # parse a key: value + w_key = self.decode_key_string(i) + i = self.skip_whitespace(self.pos) + ch = self.ll_chars[i] + if ch != ':': + self._raise("No ':' found at char %d", i) + i += 1 + + w_value = self.decode_any(i) + dict_w[w_key] = w_value + i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] i += 1 - bits |= ord(ch) - if ch == '"': + if ch == '}': self.pos = i - return self._create_string(start, i - 1, bits) - elif ch == '\\' or ch < '\x20': - self.pos = i-1 - return self.decode_string_escaped(start) + return self._create_dict(dict_w) + elif ch == ',': + i = self.skip_whitespace(i) + else: + self._raise_object_error(ch, start, i - 1) - def _create_string(self, start, end, bits): - if bits & 0x80: - # the 8th bit is set, it's an utf8 string - content_utf8 = self.getslice(start, end) + def decode_string_uncached(self, i): + start = i + ll_chars = self.ll_chars + nonascii, i = simd.find_end_of_string_no_hash(ll_chars, i, len(self.s)) + ch = ll_chars[i] + if ch == '\\': + self.pos = i + return self.decode_string_escaped(start, nonascii) + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + else: + assert ch == '"' + + self.pos = i + 1 + return self._create_string_wrapped(start, i, nonascii) + + def _create_string_wrapped(self, start, end, nonascii): + content = self.getslice(start, end) + if nonascii: + # contains non-ascii chars, we need to check that it's valid utf-8 lgt = unicodehelper.check_utf8_or_raise(self.space, - content_utf8) - return self.space.newutf8(content_utf8, lgt) + content) else: - # ascii only, fast path (ascii is a strict subset of - # latin1, and we already checked that all the chars are < - # 128) - return self.space.newutf8(self.getslice(start, end), - end - start) + lgt = end - start + return self.space.newutf8(content, lgt) def _create_dict(self, d): from pypy.objspace.std.dictmultiobject import from_unicode_key_dict @@ -313,8 +446,7 @@ from pypy.objspace.std.dictmultiobject import create_empty_unicode_key_dict return create_empty_unicode_key_dict(self.space) - - def decode_string_escaped(self, start): + def decode_string_escaped(self, start, nonascii): i = self.pos builder = StringBuilder((i - start) * 2) # just an estimate assert start >= 0 @@ -325,25 +457,21 @@ i += 1 if ch == '"': content_utf8 = builder.build() - lgt = unicodehelper.check_utf8_or_raise(self.space, + length = unicodehelper.check_utf8_or_raise(self.space, content_utf8) self.pos = i - return self.space.newutf8(content_utf8, lgt) + return self.space.newutf8(content_utf8, length) elif ch == '\\': - i = self.decode_escape_sequence(i, builder) + i = self.decode_escape_sequence_to_utf8(i, builder) elif ch < '\x20': - if ch == '\0': - self._raise("Unterminated string starting at char %d", - start - 1) - else: - self._raise("Invalid control character at char %d", i-1) + self._raise_control_char_in_string(ch, start, i) else: builder.append(ch) - def decode_escape_sequence(self, i, builder): + def decode_escape_sequence_to_utf8(self, i, stringbuilder): ch = self.ll_chars[i] i += 1 - put = builder.append + put = stringbuilder.append if ch == '\\': put('\\') elif ch == '"': put('"' ) elif ch == '/': put('/' ) @@ -353,22 +481,37 @@ elif ch == 'r': put('\r') elif ch == 't': put('\t') elif ch == 'u': - return self.decode_escape_sequence_unicode(i, builder) + # may be a suggorate pair + return self.decode_escape_sequence_unicode(i, stringbuilder) else: self._raise("Invalid \\escape: %s (char %d)", ch, i-1) return i + def _get_int_val_from_hex4(self, i): + ll_chars = self.ll_chars + res = 0 + for i in range(i, i + 4): + ch = ord(ll_chars[i]) + if ord('a') <= ch <= ord('f'): + digit = ch - ord('a') + 10 + elif ord('A') <= ch <= ord('F'): + digit = ch - ord('A') + 10 + elif ord('0') <= ch <= ord('9'): + digit = ch - ord('0') + else: + raise ValueError + res = (res << 4) + digit + return res + def decode_escape_sequence_unicode(self, i, builder): # at this point we are just after the 'u' of the \u1234 sequence. start = i i += 4 - hexdigits = self.getslice(start, i) try: - val = int(hexdigits, 16) + val = self._get_int_val_from_hex4(start) if (0xd800 <= val <= 0xdbff and self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u'): - hexdigits = self.getslice(i+2, i+6) - lowsurr = int(hexdigits, 16) + lowsurr = self._get_int_val_from_hex4(i + 2) if 0xdc00 <= lowsurr <= 0xdfff: # decode surrogate pair val = 0x10000 + (((val - 0xd800) << 10) | @@ -383,45 +526,531 @@ builder.append(utf8_ch) return i - def decode_key(self, i): - """ returns a wrapped unicode """ - from rpython.rlib.rarithmetic import intmask - i = self.skip_whitespace(i) + def decode_string(self, i, context=None): + ll_chars = self.ll_chars + start = i + ch = ll_chars[i] + if ch == '"': + self.pos = i + 1 + return self.w_empty_string # surprisingly common + + cache = True + if context is not None: + context.decoded_strings += 1 + if not context.should_cache(): + cache = False + if len(self.s) < self.MIN_SIZE_FOR_STRING_CACHE: + cache = False + + if not cache: + return self.decode_string_uncached(i) + + strhash, nonascii, i = simd.find_end_of_string(ll_chars, i, len(self.s)) + ch = ll_chars[i] + if ch == '\\': + self.pos = i + return self.decode_string_escaped(start, nonascii) + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + else: + assert ch == '"' + + self.pos = i + 1 + + length = i - start + strhash ^= length + + # check cache first: + try: + entry = self.cache_wrapped[strhash] + except KeyError: + w_res = self._create_string_wrapped(start, i, nonascii) + # only add *some* strings to the cache, because keeping them all is + # way too expensive + if (context is not None and context.decoded_strings < 200) or strhash in self.lru_cache: + entry = WrappedCacheEntry( + self.getslice(start, start + length), w_res) + self.cache_wrapped[strhash] = entry + else: + self.lru_cache[self.lru_index] = strhash + self.lru_index = (self.lru_index + 1) & self.LRU_MASK + return w_res + if not _compare_cache_entry(self.space, entry.repr, ll_chars, start, length): + # hopefully rare + return self._create_string_wrapped(start, i, nonascii) + if context is not None: + context.cache_hits += 1 + return entry.w_uni + + def decode_key(self, i, currmap): + newmap = self._decode_key(i, currmap) + currmap.observe_transition(newmap) + return newmap + + def _decode_key(self, i, currmap): + ll_chars = self.ll_chars + nextmap = currmap.fast_path_key_parse(self, i) + if nextmap is not None: + return nextmap + + start = i + ch = ll_chars[i] + if ch != '"': + self._raise("Key name must be string at char %d", i) + i += 1 + w_key = self._decode_key_string(i) + return currmap.get_next(w_key, self.s, start, self.pos) + + def _decode_key_string(self, i): + ll_chars = self.ll_chars + start = i + + strhash, nonascii, i = simd.find_end_of_string(ll_chars, i, len(self.s)) + + ch = ll_chars[i] + if ch == '\\': + self.pos = i + w_key = self.decode_string_escaped(start, nonascii) + return w_key + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + length = i - start + strhash ^= length + self.pos = i + 1 + # check cache first: + try: + entry = self.cache[strhash] + except KeyError: + w_res = self._create_string_wrapped(start, i, nonascii) + entry = WrappedCacheEntry( + self.getslice(start, start + length), w_res) + self.cache[strhash] = entry + return w_res + if not _compare_cache_entry(self.space, entry.repr, ll_chars, start, length): + # hopefully rare + w_res = self._create_string_wrapped(start, i, nonascii) + print w_res + else: + w_res = entry.w_uni + return w_res + + def decode_key_string(self, i): ll_chars = self.ll_chars ch = ll_chars[i] if ch != '"': self._raise("Key name must be string at char %d", i) i += 1 + return self._decode_key_string(i) - start = i - bits = 0 - strhash = ord(ll_chars[i]) << 7 - while True: - ch = ll_chars[i] +class WrappedCacheEntry(object): + def __init__(self, repr, w_uni): + self.repr = repr + self.w_uni = w_uni + + +class MapBase(object): + # the basic problem we are trying to solve is the following: dicts in + # json can either be used as objects, or as dictionaries with arbitrary + # string keys. We want to use maps for the former, but not for the + # latter. But we don't know in advance which kind of dict is which. + + # Therefore we create "preliminary" maps where we aren't quite sure yet + # whether they are really useful maps or not. If we see them used often + # enough, we promote them to "useful" maps, which we will actually + # instantiate objects with. + + # If we determine that a map is not used often enough, we can turn it + # into a "blocked" map, which is a point in the map tree where we will + # switch to regular dicts, when we reach that part of the tree. + + # allowed graph edges or nodes in all_next: + # USEFUL ------- + # / \ \ + # v v v + # FRINGE USEFUL BLOCKED + # | + # v + # PRELIMINARY + # | + # v + # PRELIMINARY + + # state transitions: + # PRELIMINARY + # / | \ + # | v v + # | FRINGE -> USEFUL + # | | + # \ | + # v v + # BLOCKED + + # the single_nextmap edge can only be these graph edges: + # USEFUL + # | + # v + # USEFUL + # + # FRINGE + # | + # v + # PRELIMINARY + # | + # v + # PRELIMINARY + + USEFUL = 'u' + PRELIMINARY = 'p' + FRINGE = 'f' # buffer between PRELIMINARY and USEFUL + BLOCKED = 'b' + + # tunable parameters + MAX_FRINGE = 40 + USEFUL_THRESHOLD = 5 + + def __init__(self, space): + self.space = space + + # a single transition is stored in .single_nextmap + self.single_nextmap = None + + # all_next is only initialized after seeing the *second* transition + # but then it also contains .single_nextmap + self.all_next = None # later dict {key: nextmap} + + self.instantiation_count = 0 + self.number_of_leaves = 1 + + def get_terminator(self): + while isinstance(self, JSONMap): + self = self.prev + assert isinstance(self, Terminator) + return self + + def _check_invariants(self): + if self.all_next: + for next in self.all_next.itervalues(): + next._check_invariants() + elif self.single_nextmap: + self.single_nextmap._check_invariants() + + def get_next(self, w_key, string, start, stop): + from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq + if isinstance(self, JSONMap): + assert not self.state == MapBase.BLOCKED + single_nextmap = self.single_nextmap + if (single_nextmap is not None and + single_nextmap.w_key.eq_w(w_key)): + return single_nextmap + + assert stop >= 0 + assert start >= 0 + + if single_nextmap is None: + # first transition ever seen, don't initialize all_next + next = self._make_next_map(w_key, string[start:stop]) + self.single_nextmap = next + else: + if self.all_next is None: + self.all_next = objectmodel.r_dict(unicode_eq, unicode_hash, + force_non_null=True, simple_hash_eq=True) + self.all_next[single_nextmap.w_key] = single_nextmap + else: + next = self.all_next.get(w_key, None) + if next is not None: + return next + next = self._make_next_map(w_key, string[start:stop]) + self.all_next[w_key] = next + + # fix number_of_leaves + self.change_number_of_leaves(1) + + terminator = self.get_terminator() + terminator.register_potential_fringe(next) + return next + + def change_number_of_leaves(self, difference): + parent = self + while isinstance(parent, JSONMap): + parent.number_of_leaves += difference + parent = parent.prev + parent.number_of_leaves += difference # terminator + + def fast_path_key_parse(self, decoder, position): + single_nextmap = self.single_nextmap + if single_nextmap: + ll_chars = decoder.ll_chars + assert isinstance(single_nextmap, JSONMap) + if single_nextmap.key_repr_cmp(ll_chars, position): + decoder.pos = position + len(single_nextmap.key_repr) + return single_nextmap + + def observe_transition(self, newmap): + """ observe a transition from self to newmap. + This does a few things, including updating the self size estimate with + the knowledge that one object transitioned from self to newmap. + also it potentially decides that self should move to state USEFUL.""" + self.instantiation_count += 1 + if isinstance(self, JSONMap) and self.state == MapBase.FRINGE: + if self.is_useful(): + self.mark_useful() + + def _make_next_map(self, w_key, key_repr): + return JSONMap(self.space, self, w_key, key_repr) + + def _all_dot(self, output): + identity = objectmodel.compute_unique_id(self) + output.append('%s [shape=box%s];' % (identity, self._get_dot_text())) + if self.all_next: + for w_key, value in self.all_next.items(): + assert isinstance(value, JSONMap) + if value is self.single_nextmap: + color = ", color=blue" + else: + color = "" + output.append('%s -> %s [label="%s"%s];' % ( + identity, objectmodel.compute_unique_id(value), value.w_key._utf8, color)) + value._all_dot(output) + elif self.single_nextmap is not None: + value = self.single_nextmap + output.append('%s -> %s [label="%s", color=blue];' % ( + identity, objectmodel.compute_unique_id(value), value.w_key._utf8)) + value._all_dot(output) + + + def _get_dot_text(self): + return ", label=base" + + def view(self): + from dotviewer import graphclient + import pytest + r = ["digraph G {"] + self._all_dot(r) + r.append("}") + p = pytest.ensuretemp("resilientast").join("temp.dot") + p.write("\n".join(r)) + graphclient.display_dot_file(str(p)) + + def _get_caching_stats(self): + caching = 0 + num_maps = 1 + if isinstance(self, JSONMap) and self.should_cache() and self.decoded_strings > 200: + caching += 1 + + if self.all_next: + children = self.all_next.values() + elif self.single_nextmap: + children = [self.single_nextmap] + else: + children = [] + for child in children: + a, b = child._get_caching_stats() + caching += a + num_maps += b + return caching, num_maps + +class Terminator(MapBase): + def __init__(self, space): + MapBase.__init__(self, space) + self.all_object_count = 0 + self.current_fringe = {} + + def register_potential_fringe(self, prelim): + prev = prelim.prev + if (isinstance(prev, Terminator) or + isinstance(prev, JSONMap) and prev.state == MapBase.USEFUL): + prelim.state = MapBase.FRINGE + + if len(self.current_fringe) > MapBase.MAX_FRINGE: + self.cleanup_fringe() + self.current_fringe[prelim] = None + + def cleanup_fringe(self): + min_fringe = None + min_avg = 10000000000 + for f in self.current_fringe: + if f.state == MapBase.FRINGE: + avg = f.average_instantiation() + if avg < min_avg: + min_avg = avg + min_fringe = f + else: + for f in self.current_fringe.keys(): + if f.state != MapBase.FRINGE: + del self.current_fringe[f] + return + assert min_fringe + min_fringe.mark_blocked() + del self.current_fringe[min_fringe] + + +class JSONMap(MapBase): + """ A map implementation to speed up parsing """ + + def __init__(self, space, prev, w_key, key_repr): + MapBase.__init__(self, space) + + self.prev = prev + self.w_key = w_key + self.key_repr = key_repr + + self.state = MapBase.PRELIMINARY + + # key decoding stats + self.decoded_strings = 0 + self.cache_hits = 0 + + # for jsondict support + self.key_to_index = None + self.keys_in_order = None + self.strategy_instance = None + + @jit.elidable + def get_terminator(self): + while isinstance(self, JSONMap): + self = self.prev + assert isinstance(self, Terminator) + return self + + def _check_invariants(self): + assert self.state in ( + MapBase.USEFUL, + MapBase.PRELIMINARY, + MapBase.FRINGE, + MapBase.BLOCKED, + ) + + prev = self.prev + if isinstance(prev, JSONMap): + prevstate = prev.state + else: + prevstate = MapBase.USEFUL + + if prevstate == MapBase.USEFUL: + assert self.state != MapBase.PRELIMINARY + elif prevstate == MapBase.PRELIMINARY: + assert self.state == MapBase.PRELIMINARY + elif prevstate == MapBase.FRINGE: + assert self.state == MapBase.PRELIMINARY + else: + # if prevstate is BLOCKED, we shouldn't have recursed here! + assert False, "should be unreachable" + + if self.state == MapBase.BLOCKED: + assert self.single_nextmap is None + assert self.all_next is None + + MapBase._check_invariants(self) + + def mark_useful(self): + # mark self as useful, and also the most commonly instantiated + # children, recursively + assert self.state in (MapBase.FRINGE, MapBase.PRELIMINARY) + self.state = MapBase.USEFUL + maxchild = self.single_nextmap + if self.all_next is not None: + for child in self.all_next.itervalues(): + if child.instantiation_count > maxchild.instantiation_count: + maxchild = child + if maxchild is not None: + maxchild.mark_useful() + if self.all_next: + terminator = self.get_terminator() + for child in self.all_next.itervalues(): + if child is not maxchild: + terminator.register_potential_fringe(child) + self.single_nextmap = maxchild + + def mark_blocked(self): + self.state = MapBase.BLOCKED + if self.all_next: + for next in self.all_next.itervalues(): + next.mark_blocked() + elif self.single_nextmap: + self.single_nextmap.mark_blocked() + self.single_nextmap = None + self.all_next = None + self.change_number_of_leaves(-self.number_of_leaves + 1) + + def is_blocked(self): + return self.state == MapBase.BLOCKED + + def average_instantiation(self): + return self.instantiation_count / float(self.number_of_leaves) + + def is_useful(self): + return self.average_instantiation() > self.USEFUL_THRESHOLD + + def should_cache(self): + return not (self.decoded_strings > 200 and self.cache_hits * 4 < self.decoded_strings) + + def key_repr_cmp(self, ll_chars, i): + for j, c in enumerate(self.key_repr): + if ll_chars[i] != c: + return False i += 1 - if ch == '"': - break - elif ch == '\\' or ch < '\x20': - self.pos = i-1 - return self.decode_string_escaped(start) - strhash = intmask((1000003 * strhash) ^ ord(ll_chars[i])) - bits |= ord(ch) - length = i - start - 1 - if length == 0: - strhash = -1 + return True + + # _____________________________________________________ + # methods for JsonDictStrategy + + @jit.elidable + def get_index(self, w_key): + from pypy.objspace.std.unicodeobject import W_UnicodeObject + assert isinstance(w_key, W_UnicodeObject) + return self.get_key_to_index().get(w_key, -1) + + def get_key_to_index(self): + from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq + key_to_index = self.key_to_index + if key_to_index is None: + key_to_index = self.key_to_index = objectmodel.r_dict(unicode_eq, unicode_hash, + force_non_null=True, simple_hash_eq=True) + # compute depth + curr = self + depth = 0 + while True: + depth += 1 + curr = curr.prev + if not isinstance(curr, JSONMap): + break + + curr = self + while depth: + depth -= 1 + key_to_index[curr.w_key] = depth + curr = curr.prev + if not isinstance(curr, JSONMap): + break + return key_to_index + + def get_keys_in_order(self): + keys_in_order = self.keys_in_order + if keys_in_order is None: + key_to_index = self.get_key_to_index() + keys_in_order = self.keys_in_order = [None] * len(key_to_index) + for w_key, index in key_to_index.iteritems(): + keys_in_order[index] = w_key + return keys_in_order + + # _____________________________________________________ + + def _get_dot_text(self): + if self.all_next is None: + l = int(self.single_nextmap is not None) else: - strhash ^= length - strhash = intmask(strhash) - self.pos = i - # check cache first: - key = (ll_chars, start, length, strhash) - try: - return self.cache[key] - except KeyError: - pass - res = self._create_string(start, i - 1, bits) - self.cache[key] = res + l = len(self.all_next) + 1 + extra = "" + if self.decoded_strings: + extra = "\\n%s/%s (%s%%)" % (self.cache_hits, self.decoded_strings, self.cache_hits/float(self.decoded_strings)) + res = ', label="#%s\\nchildren: %s%s"' % (self.instantiation_count, l, extra) + if self.state == MapBase.BLOCKED: + res += ", fillcolor=lightsalmon" + if self.state == MapBase.FRINGE: + res += ", fillcolor=lightgray" + if self.state == MapBase.PRELIMINARY: + res += ", fillcolor=lightslategray" return res @@ -442,3 +1071,4 @@ return w_res finally: decoder.close() + diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py new file mode 100644 --- /dev/null +++ b/pypy/module/_pypyjson/simd.py @@ -0,0 +1,218 @@ +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import objectmodel, unroll +from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT +from rpython.jit.backend.detect_cpu import autodetect + +# accelerators for string operations using simd on regular word sizes (*not* +# SSE instructions). this style is sometimes called SWAR (SIMD Within A +# Register) or "broadword techniques" + +# XXX remove wordsize and endianness restrictions properly, so far only x86-64 +# is tested + +USE_SIMD = False +if LONG_BIT == 64: + WORD_SIZE = 8 + EVERY_BYTE_ONE = 0x0101010101010101 + EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080 + if autodetect() == "x86-64": + USE_SIMD = True +else: + WORD_SIZE = 4 + EVERY_BYTE_ONE = 0x01010101 + EVERY_BYTE_HIGHEST_BIT = 0x80808080 + + +# helpers + +unrolling_wordsize = unroll.unrolling_iterable(range(WORD_SIZE)) + +def char_repeated_word_width(ch): + return r_uint(EVERY_BYTE_ONE) * ord(ch) + +def any_char_zero(word): + return (word - r_uint(EVERY_BYTE_ONE)) & ~word & r_uint(EVERY_BYTE_HIGHEST_BIT) + +def any_char_in_words_zero(*words): + return _any_char_in_any_word_zero_accum(0, *words) + +def _any_char_in_any_word_zero_accum(accum, word, *words): + accum |= (word - r_uint(EVERY_BYTE_ONE)) & ~word + if not words: + return accum & r_uint(EVERY_BYTE_HIGHEST_BIT) + return _any_char_in_any_word_zero_accum(accum, *words) + +def print_chars(word): + # for debugging + out = '' + for i in range(WORD_SIZE): + out += chr(word & 0xff) + word >>= 8 + return out + +def index_nonzero(word): + # XXX can be done very cheap in theory + assert word + for i in unrolling_wordsize: + if word & 0xff: + return i + word >>= 8 + assert 0 + +def index_zero(word): + # XXX can be done very cheap in theory + assert any_char_zero(word) + for i in unrolling_wordsize: + if not word & 0xff: + return i + word >>= 8 + assert 0 # XXX ??? + +def splice_words(word, offset, other): + mask = ((~r_uint(0)) << (8 * offset)) + return (word & mask) | (other & ~mask) + + + + at objectmodel.always_inline +def position_string_ender(word): + maskquote = char_repeated_word_width('"') + maskbackslash = char_repeated_word_width('\\') + maskx20 = char_repeated_word_width(chr(0xff - 0x1f)) + # x1 and x2 check for equality, if a byte is 0 the corresponding + # char is equal to " or \ + x1 = maskquote ^ word + x2 = maskbackslash ^ word + # x3 checks for char < 0x20, the byte is 0 in that case + x3 = maskx20 & word + return any_char_in_words_zero(x1, x2, x3) + + at objectmodel.always_inline +def find_end_of_string_simd_unaligned(ll_chars, startpos, length): + ch = ll_chars[startpos] + strhash = (ord(ch) << 7) ^ 0x345678 + + wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) + num_safe_reads = (length - startpos) // WORD_SIZE + + bits = 0 + for i in range(num_safe_reads): + word = wordarray[i] + cond = position_string_ender(word) + if cond: + break + bits |= word + strhash = intmask((1000003 * strhash) ^ intmask(word)) + else: + # didn't find end of string yet, look at remaining chars + word = 0 + shift = 0 + i = 0 + for i in range(num_safe_reads * WORD_SIZE + startpos, length + 1): + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + bits |= ord(ch) + word |= ord(ch) << shift + shift += 8 + if shift: + strhash = intmask((1000003 * strhash) ^ intmask(word)) + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + return strhash, nonascii, i + + # compute endposition + nonzero = index_nonzero(cond) + endposition = startpos + i * WORD_SIZE + nonzero + if nonzero: + word = splice_words(r_uint(0), nonzero, word) + bits |= word + strhash = intmask((1000003 * strhash) ^ intmask(word)) + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + + return strhash, nonascii, endposition + + at objectmodel.always_inline +def find_end_of_string_simd_unaligned_no_hash(ll_chars, startpos, length): + ch = ll_chars[startpos] + + wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) + num_safe_reads = (length - startpos) // WORD_SIZE + + bits = 0 + for i in range(num_safe_reads): + word = wordarray[i] + cond = position_string_ender(word) + if cond: + break + bits |= word + else: + # didn't find end of string yet, look at remaining chars + word = 0 + shift = 0 + i = 0 + for i in range(num_safe_reads * WORD_SIZE + startpos, length + 1): + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + bits |= ord(ch) + word |= ord(ch) << shift + shift += WORD_SIZE + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + return nonascii, i + + # compute endposition + nonzero = index_nonzero(cond) + endposition = startpos + i * WORD_SIZE + nonzero + if nonzero: + word = splice_words(r_uint(0), nonzero, word) + bits |= word + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + + return nonascii, endposition + + + at objectmodel.always_inline +def find_end_of_string_slow(ll_chars, i, length): + ch = ll_chars[i] + strhash = (ord(ch) << 7) ^ 0x345678 + word = 0 + shift = 0 + + bits = 0 + + while True: + # this loop is a fast path for strings which do not contain escape + # characters + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + i += 1 + bits |= ord(ch) + + word |= ord(ch) << shift + shift += WORD_SIZE + if shift == WORD_SIZE * WORD_SIZE: + strhash = intmask((1000003 * strhash) ^ word) + shift = 0 + word = 0 + + if shift: + strhash = intmask((1000003 * strhash) ^ word) + return strhash, bool(bits & 0x80), i + +if USE_SIMD: + find_end_of_string = find_end_of_string_simd_unaligned + find_end_of_string_no_hash = find_end_of_string_simd_unaligned_no_hash +else: + find_end_of_string = find_end_of_string_slow + + @objectmodel.always_inline + def find_end_of_string_no_hash(ll_chars, i, length): + _, nonascii, i = find_end_of_string_slow(ll_chars, i, length) + return (nonascii, i) + + diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -1,31 +1,253 @@ # -*- encoding: utf-8 -*- -from pypy.module._pypyjson.interp_decoder import JSONDecoder +import pytest +from pypy.module._pypyjson.interp_decoder import JSONDecoder, Terminator, MapBase +from rpython.rtyper.lltypesystem import lltype, rffi -def test_skip_whitespace(): - s = ' hello ' - dec = JSONDecoder('fake space', s) - assert dec.pos == 0 - assert dec.skip_whitespace(0) == 3 - assert dec.skip_whitespace(3) == 3 - assert dec.skip_whitespace(8) == len(s) - dec.close() -class FakeSpace(object): - def newutf8(self, s, l): - return s +class TestJson(object): + def test_skip_whitespace(self): + s = ' hello ' + dec = JSONDecoder(self.space, s) + assert dec.pos == 0 + assert dec.skip_whitespace(0) == 3 + assert dec.skip_whitespace(3) == 3 + assert dec.skip_whitespace(8) == len(s) + dec.close() -def test_decode_key(): - s1 = "123" * 100 - s = ' "%s" "%s" ' % (s1, s1) - dec = JSONDecoder(FakeSpace(), s) - assert dec.pos == 0 - x = dec.decode_key(0) - assert x == s1 - # check caching - y = dec.decode_key(dec.pos) - assert y == s1 - assert y is x - dec.close() + def test_json_map(self): + m = Terminator(self.space) + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + m1 = m.get_next(w_a, '"a"', 0, 3) + assert m1.w_key == w_a + assert m1.single_nextmap is None + assert m1.key_repr == '"a"' + assert m1.key_repr_cmp('"a": 123', 0) + assert not m1.key_repr_cmp('b": 123', 0) + assert m.single_nextmap.w_key == w_a + + m2 = m.get_next(w_a, '"a"', 0, 3) + assert m2 is m1 + + m3 = m.get_next(w_b, '"b"', 0, 3) + assert m3.w_key == w_b + assert m3.single_nextmap is None + assert m3.key_repr == '"b"' + assert m.single_nextmap is m1 + + m4 = m3.get_next(w_c, '"c"', 0, 3) + assert m4.w_key == w_c + assert m4.single_nextmap is None + assert m4.key_repr == '"c"' + assert m3.single_nextmap is m4 + + def test_json_map_get_index(self): + m = Terminator(self.space) + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + m1 = m.get_next(w_a, 'a"', 0, 2) + assert m1.get_index(w_a) == 0 + assert m1.get_index(w_b) == -1 + + m2 = m.get_next(w_b, 'b"', 0, 2) + assert m2.get_index(w_b) == 0 + assert m2.get_index(w_a) == -1 + + m3 = m2.get_next(w_c, 'c"', 0, 2) + assert m3.get_index(w_b) == 0 + assert m3.get_index(w_c) == 1 + assert m3.get_index(w_a) == -1 + + def test_decode_key(self): + m = Terminator(self.space) + m_diff = Terminator(self.space) + for s1 in ["abc", "1001" * 10, u"ä".encode("utf-8")]: + s = ' "%s" "%s" "%s"' % (s1, s1, s1) + dec = JSONDecoder(self.space, s) + assert dec.pos == 0 + m1 = dec.decode_key(dec.skip_whitespace(0), m) + assert m1.w_key._utf8 == s1 + assert m1.key_repr == '"%s"' % s1 + + # check caching on w_key level + m2 = dec.decode_key(dec.skip_whitespace(dec.pos), m_diff) + assert m1.w_key is m2.w_key + + # check caching on map level + m3 = dec.decode_key(dec.skip_whitespace(dec.pos), m_diff) + assert m3 is m2 + dec.close() + + def test_decode_string_caching(self): + for s1 in ["abc", u"ä".encode("utf-8")]: + s = '"%s" "%s" "%s"' % (s1, s1, s1) + dec = JSONDecoder(self.space, s) + dec.MIN_SIZE_FOR_STRING_CACHE = 0 + assert dec.pos == 0 + w_x = dec.decode_string(1) + w_y = dec.decode_string(dec.skip_whitespace(dec.pos) + 1) + assert w_x is not w_y + # check caching + w_z = dec.decode_string(dec.skip_whitespace(dec.pos) + 1) + assert w_z is w_y + dec.close() + + def _make_some_maps(self): + # base -> m1 -> m2 -> m3 + # \-> m4 + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + w_d = self.space.newutf8("d", 1) + base = Terminator(self.space) + base.instantiation_count = 6 + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = m1.get_next(w_b, 'b"', 0, 2) + m3 = m2.get_next(w_c, 'c"', 0, 2) + m4 = m2.get_next(w_d, 'd"', 0, 2) + return base, m1, m2, m3, m4 + + # unit tests for map state transistions + def test_fringe_to_useful(self): + base, m1, m2, m3, m4 = self._make_some_maps() + base.instantiation_count = 6 + assert m1.state == MapBase.FRINGE + m1.instantiation_count = 6 + + assert m2.state == MapBase.PRELIMINARY + m2.instantiation_count = 6 + + assert m3.state == MapBase.PRELIMINARY + m3.instantiation_count = 2 + assert m2.single_nextmap is m3 + + assert m4.state == MapBase.PRELIMINARY + m4.instantiation_count = 4 + + m1.mark_useful() + assert m1.state == MapBase.USEFUL + assert m2.state == MapBase.USEFUL + assert m3.state == MapBase.FRINGE + assert m4.state == MapBase.USEFUL + assert m2.single_nextmap is m4 + + assert m1.number_of_leaves == 2 + base._check_invariants() + + def test_number_of_leaves(self): + w_x = self.space.newutf8("x", 1) + base, m1, m2, m3, m4 = self._make_some_maps() + assert base.number_of_leaves == 2 + assert m1.number_of_leaves == 2 + assert m2.number_of_leaves == 2 + assert m3.number_of_leaves == 1 + assert m4.number_of_leaves == 1 + m5 = m2.get_next(w_x, 'x"', 0, 2) + assert base.number_of_leaves == 3 + assert m1.number_of_leaves == 3 + assert m2.number_of_leaves == 3 + assert m5.number_of_leaves == 1 + + def test_cleanup_fringe_simple(self): + base, m1, m2, m3, m4 = self._make_some_maps() + base.instantiation_count = 6 + assert m1.state == MapBase.FRINGE + m1.instantiation_count = 6 + m2.instantiation_count = 6 + m3.instantiation_count = 2 + m4.instantiation_count = 4 + assert base.current_fringe == {m1: None} + + m1.mark_useful() + assert base.current_fringe == {m1: None, m3: None} # not cleaned up + base.cleanup_fringe() + assert base.current_fringe == {m3: None} + + def test_cleanup_fringe_block(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + w_d = self.space.newutf8("d", 1) + base = Terminator(self.space) + base.instantiation_count = 6 + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = base.get_next(w_b, 'b"', 0, 2) + m3 = base.get_next(w_c, 'c"', 0, 2) + m4 = base.get_next(w_d, 'd"', 0, 2) + m5 = m4.get_next(w_a, 'a"', 0, 2) + base.instantiation_count = 7 + m1.instantiation_count = 2 + m2.instantiation_count = 2 + m3.instantiation_count = 2 + m4.instantiation_count = 1 + m5.instantiation_count = 1 + assert base.current_fringe == dict.fromkeys([m1, m2, m3, m4]) + + base.cleanup_fringe() + assert base.current_fringe == dict.fromkeys([m1, m2, m3]) + assert m4.state == MapBase.BLOCKED + assert m4.single_nextmap is None + assert m4.all_next is None + assert m5.state == MapBase.BLOCKED + assert m5.single_nextmap is None + assert m5.all_next is None + + def test_deal_with_blocked(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + space = self.space + s = '{"a": 1, "b": 2, "c": 3}' + dec = JSONDecoder(space, s) + dec.startmap = base = Terminator(space) + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = m1.get_next(w_b, 'b"', 0, 2) + m2.mark_blocked() + w_res = dec.decode_object(1) + assert space.int_w(space.len(w_res)) == 3 + assert space.int_w(space.getitem(w_res, w_a)) == 1 + assert space.int_w(space.getitem(w_res, w_b)) == 2 + assert space.int_w(space.getitem(w_res, w_c)) == 3 + dec.close() + + def test_deal_with_blocked_number_of_leaves(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_x = self.space.newutf8("x", 1) + w_u = self.space.newutf8("u", 1) + space = self.space + base = Terminator(space) + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = m1.get_next(w_b, 'b"', 0, 2) + m2.get_next(w_x, 'x"', 0, 2) + m2.get_next(w_u, 'u"', 0, 2) + assert base.number_of_leaves == 2 + m2.mark_blocked() + assert base.number_of_leaves == 1 + + @pytest.mark.skip() + def test_caching_stats(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_x = self.space.newutf8("x", 1) + w_u = self.space.newutf8("u", 1) + space = self.space + base = Terminator(space) + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = m1.get_next(w_b, 'b"', 0, 2) + m2.get_next(w_x, 'x"', 0, 2) + m2.get_next(w_u, 'u"', 0, 2) + m1.decode_string = 300 + m1.cache_hits = 0 + m3 = base.get_next(w_b, '"b"', 0, 3) + m3.decode_string = 300 + m3.cache_hits = 300 + caching_maps, total_maps = base._get_caching_stats() + assert caching_maps == 5 + assert total_maps == 6 + class AppTest(object): spaceconfig = {"objspace.usemodules._pypyjson": True} @@ -55,7 +277,7 @@ raises(ValueError, _pypyjson.loads, 'fa') raises(ValueError, _pypyjson.loads, 'f') raises(ValueError, _pypyjson.loads, 'falXX') - + def test_decode_string(self): import _pypyjson @@ -85,7 +307,7 @@ import _pypyjson assert _pypyjson.loads(r'"\\"') == u'\\' assert _pypyjson.loads(r'"\""') == u'"' - assert _pypyjson.loads(r'"\/"') == u'/' + assert _pypyjson.loads(r'"\/"') == u'/' assert _pypyjson.loads(r'"\b"') == u'\b' assert _pypyjson.loads(r'"\f"') == u'\f' assert _pypyjson.loads(r'"\n"') == u'\n' @@ -101,12 +323,19 @@ import _pypyjson s = r'"hello\nworld' # missing the trailing " raises(ValueError, "_pypyjson.loads(s)") - + def test_escape_sequence_unicode(self): import _pypyjson s = r'"\u1234"' assert _pypyjson.loads(s) == u'\u1234' + def test_escape_sequence_mixed_with_utf8(self): + import _pypyjson + utf8 = u'ä"'.encode("utf-8") + assert _pypyjson.loads(r'"abc\\' + utf8) == u'abc\\ä' + assert _pypyjson.loads(r'"abc\"' + utf8) == u'abc"ä' + assert _pypyjson.loads(r'"def\u1234' + utf8) == u'def\u1234ä' + def test_invalid_utf_8(self): import _pypyjson s = '"\xe0"' # this is an invalid UTF8 sequence inside a string @@ -176,13 +405,18 @@ s = '{"hello": "world", "aaa": "bbb"}' assert _pypyjson.loads(s) == {'hello': 'world', 'aaa': 'bbb'} + assert _pypyjson.loads(s) == {'hello': 'world', + 'aaa': 'bbb'} raises(ValueError, _pypyjson.loads, '{"key"') raises(ValueError, _pypyjson.loads, '{"key": 42') + assert _pypyjson.loads('{"neighborhood": ""}') == { + "neighborhood": ""} + def test_decode_object_nonstring_key(self): import _pypyjson raises(ValueError, "_pypyjson.loads('{42: 43}')") - + def test_decode_array(self): import _pypyjson assert _pypyjson.loads('[]') == [] @@ -263,3 +497,4 @@ for inputtext, errmsg in test_cases: exc = raises(ValueError, _pypyjson.loads, inputtext) assert str(exc.value) == errmsg + diff --git a/pypy/module/_pypyjson/test/test_simd.py b/pypy/module/_pypyjson/test/test_simd.py new file mode 100644 --- /dev/null +++ b/pypy/module/_pypyjson/test/test_simd.py @@ -0,0 +1,104 @@ +import sys +import pytest +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import r_uint, intmask + +from pypy.module._pypyjson.simd import USE_SIMD +from pypy.module._pypyjson.simd import find_end_of_string_slow +from pypy.module._pypyjson.simd import print_chars +from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned, WORD_SIZE +from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned_no_hash + +from hypothesis import example, given, strategies + +if not USE_SIMD: + pytest.skip("only implemented for 64 bit for now") + +def fill_to_word_size(res, ch=" "): + if len(res) % WORD_SIZE != 0: + res += ch * (WORD_SIZE - (len(res) % WORD_SIZE)) + return res + +def string_to_word(s): + assert len(s) == WORD_SIZE + ll_chars, flag = rffi.get_nonmovingbuffer_final_null(s) + try: + wordarray = rffi.cast(rffi.ULONGP, ll_chars) + return wordarray[0] + finally: + rffi.free_nonmovingbuffer(s, ll_chars, flag) + +def ll(callable, string, *args): + ll_chars, flag = rffi.get_nonmovingbuffer_final_null(string) + try: + return callable(ll_chars, *args) + finally: + rffi.free_nonmovingbuffer(string, ll_chars, flag) + +word = strategies.builds( + r_uint, strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint)) + +def build_string(prefix, content, end, suffix): + res = prefix + '"' + "".join([chr(x) for x in content]) + end + suffix + return fill_to_word_size(res), len(prefix) + 1 + +string_in_context_strategy = strategies.builds( + build_string, prefix=strategies.binary(), + content=strategies.lists(strategies.integers(1, 255), min_size=1), + end=strategies.sampled_from('"\\\x00\x01'), + suffix=strategies.binary()) + +def compare(string, res1, res2): + hash1, nonascii1, endindex1 = res1 + hash2, nonascii2, endindex2 = res2 + assert endindex1 == endindex2 + if string[endindex1 - 1] == '"': + assert hash1 == hash2 + assert nonascii1 == nonascii2 + + + at example(('" \x80" ', 1)) + at example(('"\x01" ', 1)) + at example(('"aaaaaaaa"\x00\x00\x00\x00\x00\x00\x00 ', 1)) + at example(('"aaaaaaaa" ', 1)) + at example(('"12"', 1)) + at example(('"1234567abcdefghAB"', 1)) + at example(('"1234567abcdefgh"', 1)) + at example((' "123456ABCDEF" \x00', 2)) + at example((' "123456aaaaaaaaABCDEF"\x00', 2)) + at given(string_in_context_strategy) +def test_find_end_of_string(a): + (string, startindex) = a + res = ll(find_end_of_string_slow, string, startindex, len(string)) + hash, nonascii1, endposition1 = res + ch = string[endposition1] + assert ch == '"' or ch == '\\' or ch < '\x20' + for ch in string[startindex:endposition1]: + assert not (ch == '"' or ch == '\\' or ch < '\x20') + compare(string, res, ll(find_end_of_string_simd_unaligned, string, startindex, len(string))) + + nonascii2, endposition2 = ll(find_end_of_string_simd_unaligned_no_hash, string, startindex, len(string)) + assert nonascii1 == nonascii2 + assert endposition1 == endposition2 + + at given(string_in_context_strategy, strategies.binary(min_size=1)) +def test_find_end_of_string_position_invariance(a, prefix): + fn = find_end_of_string_simd_unaligned + (string, startindex) = a + h1, nonascii1, i1 = ll(fn, string, startindex, len(string)) + string2 = prefix + string + h2, nonascii2, i2 = ll(fn, string2, startindex + len(prefix), len(string) + len(prefix)) + assert h1 == h2 + assert nonascii1 == nonascii2 + assert i1 + len(prefix) == i2 + + at given(string_in_context_strategy, strategies.binary(min_size=1)) +def test_find_end_of_string_position_invariance_no_hash(a, prefix): + fn = find_end_of_string_simd_unaligned_no_hash + (string, startindex) = a + nonascii1, i1 = ll(fn, string, startindex, len(string)) + string2 = prefix + string + nonascii2, i2 = ll(fn, string2, startindex + len(prefix), len(string) + len(prefix)) + assert nonascii1 == nonascii2 + assert i1 + len(prefix) == i2 + diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py new file mode 100644 --- /dev/null +++ b/pypy/objspace/std/jsondict.py @@ -0,0 +1,167 @@ +"""dict implementation specialized for object loaded by the _pypyjson module. + +Somewhat similar to MapDictStrategy, also uses a map. +""" + +from rpython.rlib import jit, rerased, objectmodel, debug + +from pypy.objspace.std.dictmultiobject import ( + UnicodeDictStrategy, DictStrategy, + create_iterator_classes, W_DictObject) + + +def from_values_and_jsonmap(space, values_w, jsonmap): + if not objectmodel.we_are_translated(): + assert len(values_w) == len(jsonmap.get_keys_in_order()) + assert len(values_w) != 0 + debug.make_sure_not_resized(values_w) + strategy = jsonmap.strategy_instance + if strategy is None: + jsonmap.strategy_instance = strategy = JsonDictStrategy(space, jsonmap) + storage = strategy.erase(values_w) + return W_DictObject(space, strategy, storage) + +def devolve_jsonmap_dict(w_dict): + assert isinstance(w_dict, W_DictObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, JsonDictStrategy) + strategy.switch_to_unicode_strategy(w_dict) + +def get_jsonmap_from_dict(w_dict): + assert isinstance(w_dict, W_DictObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, JsonDictStrategy) + return strategy.jsonmap + +class JsonDictStrategy(DictStrategy): + erase, unerase = rerased.new_erasing_pair("jsondict") + erase = staticmethod(erase) + unerase = staticmethod(unerase) + + _immutable_fields_ = ['jsonmap'] + + def __init__(self, space, jsonmap): + DictStrategy.__init__(self, space) + self.jsonmap = jsonmap + + def wrap(self, w_key): + return w_key + + def wrapkey(space, key): + return key + + def get_empty_storage(self): + raise NotImplementedError("should not be reachable") + + def is_correct_type(self, w_obj): + space = self.space + return space.is_w(space.type(w_obj), space.w_unicode) + + def _never_equal_to(self, w_lookup_type): + return False + + def length(self, w_dict): + return len(self.unerase(w_dict.dstorage)) + + def getitem(self, w_dict, w_key): + if self.is_correct_type(w_key): + return self.getitem_unicode(w_dict, w_key) + else: + self.switch_to_unicode_strategy(w_dict) + return w_dict.getitem(w_key) + + def getitem_unicode(self, w_dict, w_key): + storage_w = self.unerase(w_dict.dstorage) + if jit.isconstant(w_key): + jit.promote(self) + index = self.jsonmap.get_index(w_key) + if index == -1: + return None + return storage_w[index] + + def setitem(self, w_dict, w_key, w_value): + if self.is_correct_type(w_key): + storage_w = self.unerase(w_dict.dstorage) + index = self.jsonmap.get_index(w_key) + if index != -1: + storage_w[index] = w_value + return + self.switch_to_unicode_strategy(w_dict) + w_dict.setitem(w_key, w_value) + + def setdefault(self, w_dict, w_key, w_default): + if self.is_correct_type(w_key): + w_result = self.getitem_unicode(w_dict, w_key) + if w_result is not None: + return w_result + self.switch_to_unicode_strategy(w_dict) + return w_dict.setdefault(w_key, w_default) + + def delitem(self, w_dict, w_key): + self.switch_to_unicode_strategy(w_dict) + return w_dict.delitem(w_key) + + def popitem(self, w_dict): + self.switch_to_unicode_strategy(w_dict) + return w_dict.popitem() + + def switch_to_unicode_strategy(self, w_dict): + strategy = self.space.fromcache(UnicodeDictStrategy) + values_w = self.unerase(w_dict.dstorage) + storage = strategy.get_empty_storage() + d_new = strategy.unerase(storage) + keys_in_order = self.jsonmap.get_keys_in_order() + assert len(keys_in_order) == len(values_w) + for index, w_key in enumerate(keys_in_order): + assert w_key is not None + assert type(w_key) is self.space.UnicodeObjectCls + d_new[w_key] = values_w[index] + w_dict.set_strategy(strategy) + w_dict.dstorage = storage + + def w_keys(self, w_dict): + return self.space.newlist(self.jsonmap.get_keys_in_order()) + + def values(self, w_dict): + return self.unerase(w_dict.dstorage)[:] # to make resizable + + def items(self, w_dict): + space = self.space + storage_w = self.unerase(w_dict.dstorage) + res = [None] * len(storage_w) + for index, w_key in enumerate(self.jsonmap.get_keys_in_order()): + res[index] = space.newtuple([w_key, storage_w[index]]) + return res + + def getiterkeys(self, w_dict): + return iter(self.jsonmap.get_keys_in_order()) + + def getitervalues(self, w_dict): + storage_w = self.unerase(w_dict.dstorage) + return iter(storage_w) + + def getiteritems_with_hash(self, w_dict): + storage_w = self.unerase(w_dict.dstorage) + return ZipItemsWithHash(self.jsonmap.get_keys_in_order(), storage_w) + + +class ZipItemsWithHash(object): + def __init__(self, list1, list2): + assert len(list1) == len(list2) + self.list1 = list1 + self.list2 = list2 + self.i = 0 + + def __iter__(self): + return self + + def next(self): + i = self.i + if i >= len(self.list1): + raise StopIteration + self.i = i + 1 + w_key = self.list1[i] + return (w_key, self.list2[i], w_key.hash_w()) + + +create_iterator_classes(JsonDictStrategy) diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py new file mode 100644 --- /dev/null +++ b/pypy/objspace/std/test/test_jsondict.py @@ -0,0 +1,89 @@ + +class AppTest(object): + spaceconfig = {"objspace.usemodules._pypyjson": True} + + def test_check_strategy(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1}') + assert __pypy__.strategy(d) == "JsonDictStrategy" + d = _pypyjson.loads('{}') + assert __pypy__.strategy(d) == "EmptyDictStrategy" + + def test_simple(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert len(d) == 2 + assert d[u"a"] == 1 + assert d[u"b"] == u"x" + assert u"c" not in d + + d[u"a"] = 5 + assert d[u"a"] == 5 + assert __pypy__.strategy(d) == "JsonDictStrategy" + + # devolve it + assert not 1 in d + assert __pypy__.strategy(d) == "UnicodeDictStrategy" + assert len(d) == 2 + assert d[u"a"] == 5 + assert d[u"b"] == u"x" + assert u"c" not in d + + def test_setdefault(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert d.setdefault(u"a", "blub") == 1 + d.setdefault(u"x", 23) + assert __pypy__.strategy(d) == "UnicodeDictStrategy" + assert len(d) == 3 + assert d == {u"a": 1, u"b": "x", u"x": 23} + + def test_delitem(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + del d[u"a"] + assert __pypy__.strategy(d) == "UnicodeDictStrategy" + assert len(d) == 1 + assert d == {u"b": "x"} + + def test_popitem(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + k, v = d.popitem() + assert __pypy__.strategy(d) == "UnicodeDictStrategy" + if k == u"a": + assert v == 1 + assert len(d) == 1 + assert d == {u"b": "x"} + else: + assert v == u"x" + assert len(d) == 1 + assert d == {u"a": 1} + + def test_keys_value_items(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert d.keys() == [u"a", u"b"] + assert d.values() == [1, u"x"] + assert d.items() == [(u"a", 1), (u"b", u"x")] + + def test_iter_keys_value_items(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert list(d.iterkeys()) == [u"a", u"b"] + assert list(d.itervalues()) == [1, u"x"] + assert list(d.iteritems()) == [(u"a", 1), (u"b", u"x")] From pypy.commits at gmail.com Wed Jun 5 08:51:26 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 05:51:26 -0700 (PDT) Subject: [pypy-commit] pypy default: Backed out changeset cefcc54d57f7 Message-ID: <5cf7bace.1c69fb81.d515a.aaa0@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r96755:f4e45930f401 Date: 2019-06-05 14:50 +0200 http://bitbucket.org/pypy/pypy/changeset/f4e45930f401/ Log: Backed out changeset cefcc54d57f7 diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -1,13 +1,11 @@ import sys from rpython.rlib.rstring import StringBuilder -from rpython.rlib.objectmodel import specialize, always_inline -from rpython.rlib import rfloat, runicode, jit, objectmodel, rutf8 +from rpython.rlib.objectmodel import specialize, always_inline, r_dict +from rpython.rlib import rfloat, rutf8 from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import r_uint from pypy.interpreter.error import oefmt from pypy.interpreter import unicodehelper -from pypy.interpreter.baseobjspace import W_Root -from pypy.module._pypyjson import simd OVF_DIGITS = len(str(sys.maxint)) @@ -17,85 +15,45 @@ # precomputing negative powers of 10 is MUCH faster than using e.g. math.pow # at runtime NEG_POW_10 = [10.0**-i for i in range(16)] -del i - def neg_pow_10(x, exp): if exp >= len(NEG_POW_10): return 0.0 return x * NEG_POW_10[exp] -def _compare_cache_entry(space, res, ll_chars, start, length): - if length != len(res): +def slice_eq(a, b): + (ll_chars1, start1, length1, _) = a + (ll_chars2, start2, length2, _) = b + if length1 != length2: return False - index = start - for c in res: - x = ord(c) - if not ll_chars[index] == chr(x): + j = start2 + for i in range(start1, start1 + length1): + if ll_chars1[i] != ll_chars2[j]: return False - index += 1 + j += 1 return True +def slice_hash(a): + (ll_chars, start, length, h) = a + return h -class IntCache(object): - START = -10 - END = 256 - - def __init__(self, space): - self.space = space - self.cache = [self.space.newint(i) - for i in range(self.START, self.END)] - - def newint(self, intval): - if self.START <= intval < self.END: - return self.cache[intval - self.START] - return self.space.newint(intval) - - -class JSONDecoder(W_Root): - - LRU_SIZE = 16 - LRU_MASK = LRU_SIZE - 1 - - DEFAULT_SIZE_SCRATCH = 20 - - MIN_SIZE_FOR_STRING_CACHE = 1024 * 1024 - - +TYPE_UNKNOWN = 0 +TYPE_STRING = 1 +class JSONDecoder(object): def __init__(self, space, s): self.space = space - self.w_empty_string = space.newutf8("", 0) - self.s = s - # we put our string in a raw buffer so: # 1) we automatically get the '\0' sentinel at the end of the string, # which means that we never have to check for the "end of string" # 2) we can pass the buffer directly to strtod - self.ll_chars, self.flag = rffi.get_nonmovingbuffer_final_null(self.s) + self.ll_chars = rffi.str2charp(s) self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') self.pos = 0 - self.intcache = space.fromcache(IntCache) - - self.cache = {} - self.cache_wrapped = {} - - self.lru_cache = [0] * self.LRU_SIZE - self.lru_index = 0 - - self.startmap = self.space.fromcache(Terminator) - self.unclear_objects = [] - - self.scratch = [[None] * self.DEFAULT_SIZE_SCRATCH] # list of scratch space - + self.cache = r_dict(slice_eq, slice_hash, simple_hash_eq=True) def close(self): - rffi.free_nonmovingbuffer(self.s, self.ll_chars, self.flag) + rffi.free_charp(self.ll_chars) lltype.free(self.end_ptr, flavor='raw') - # clean up objects that are instances of now blocked maps - for w_obj in self.unclear_objects: - jsonmap = self._get_jsonmap_from_dict(w_obj) - if jsonmap.is_blocked(): - self._devolve_jsonmap_dict(w_obj) def getslice(self, start, end): assert start >= 0 @@ -103,15 +61,18 @@ return self.s[start:end] def skip_whitespace(self, i): - ll_chars = self.ll_chars while True: - ch = ll_chars[i] + ch = self.ll_chars[i] if is_whitespace(ch): - i += 1 + i+=1 else: break return i + @specialize.arg(1) + def _raise(self, msg, *args): + raise oefmt(self.space.w_ValueError, msg, *args) + def decode_any(self, i): i = self.skip_whitespace(i) ch = self.ll_chars[i] @@ -141,11 +102,6 @@ self._raise("No JSON object could be decoded: unexpected '%s' at char %d", ch, i) - - @specialize.arg(1) - def _raise(self, msg, *args): - raise oefmt(self.space.w_ValueError, msg, *args) - def decode_null(self, i): if (self.ll_chars[i] == 'u' and self.ll_chars[i+1] == 'l' and @@ -206,7 +162,7 @@ return self.decode_int_slow(start) self.pos = i - return self.intcache.newint(intval) + return self.space.newint(intval) def decode_float(self, i): from rpython.rlib import rdtoa @@ -258,26 +214,6 @@ ovf_maybe = (count >= OVF_DIGITS) return i, ovf_maybe, sign * intval - def _raise_control_char_in_string(self, ch, startindex, currindex): - if ch == '\0': - self._raise("Unterminated string starting at char %d", - startindex - 1) - else: - self._raise("Invalid control character at char %d", currindex-1) - - def _raise_object_error(self, ch, start, i): - if ch == '\0': - self._raise("Unterminated object starting at char %d", start) - else: - self._raise("Unexpected '%s' when decoding object (char %d)", - ch, i) - - def decode_surrogate_pair(self, i, highsurr): - """ uppon enter the following must hold: - chars[i] == "\\" and chars[i+1] == "u" - """ - # the possible ValueError is caught by the caller - def decode_array(self, i): w_list = self.space.newlist([]) start = i @@ -304,13 +240,6 @@ self._raise("Unexpected '%s' when decoding array (char %d)", ch, i-1) - def decode_any_context(self, i, context): - i = self.skip_whitespace(i) - ch = self.ll_chars[i] - if ch == '"': - return self.decode_string(i+1, context) - return self.decode_any(i) - def decode_object(self, i): start = i @@ -319,124 +248,62 @@ self.pos = i+1 return self.space.newdict() - if self.scratch: - values_w = self.scratch.pop() - else: - values_w = [None] * self.DEFAULT_SIZE_SCRATCH - nextindex = 0 - currmap = self.startmap + d = self._create_empty_dict() while True: # parse a key: value - currmap = self.decode_key(i, currmap) + w_name = self.decode_key(i) i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] if ch != ':': self._raise("No ':' found at char %d", i) i += 1 - - w_value = self.decode_any_context(i, currmap) - - if nextindex == len(values_w): # full - values_w = values_w + [None] * len(values_w) # double - values_w[nextindex] = w_value - nextindex += 1 + i = self.skip_whitespace(i) + # + w_value = self.decode_any(i) + d[w_name] = w_value i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] i += 1 if ch == '}': self.pos = i - if currmap.is_blocked(): - currmap.instantiation_count += 1 - self.scratch.append(values_w) # can reuse next time - dict_w = self._switch_to_dict(currmap, values_w, nextindex) - return self._create_dict(dict_w) - self.scratch.append(values_w) # can reuse next time - values_w = values_w[:nextindex] - currmap.instantiation_count += 1 - w_res = self._create_dict_map(values_w, currmap) - if currmap.state != MapBase.USEFUL: - self.unclear_objects.append(w_res) - return w_res + return self._create_dict(d) elif ch == ',': - i = self.skip_whitespace(i) - if currmap.is_blocked(): - currmap.instantiation_count += 1 - self.scratch.append(values_w) # can reuse next time - dict_w = self._switch_to_dict(currmap, values_w, nextindex) - return self.decode_object_dict(i, start, dict_w) + pass + elif ch == '\0': + self._raise("Unterminated object starting at char %d", start) else: - self._raise_object_error(ch, start, i - 1) + self._raise("Unexpected '%s' when decoding object (char %d)", + ch, i-1) - def _create_dict_map(self, values_w, jsonmap): - from pypy.objspace.std.jsondict import from_values_and_jsonmap - return from_values_and_jsonmap(self.space, values_w, jsonmap) - - def _devolve_jsonmap_dict(self, w_dict): - from pypy.objspace.std.jsondict import devolve_jsonmap_dict - devolve_jsonmap_dict(w_dict) - - def _get_jsonmap_from_dict(self, w_dict): - from pypy.objspace.std.jsondict import get_jsonmap_from_dict - return get_jsonmap_from_dict(w_dict) - - def _switch_to_dict(self, currmap, values_w, nextindex): - dict_w = self._create_empty_dict() - index = nextindex - 1 - while isinstance(currmap, JSONMap): - dict_w[currmap.w_key] = values_w[index] - index -= 1 - currmap = currmap.prev - assert len(dict_w) == nextindex - return dict_w - - def decode_object_dict(self, i, start, dict_w): + def decode_string(self, i): + start = i + bits = 0 while True: - # parse a key: value - w_key = self.decode_key_string(i) - i = self.skip_whitespace(self.pos) - ch = self.ll_chars[i] - if ch != ':': - self._raise("No ':' found at char %d", i) - i += 1 - - w_value = self.decode_any(i) - dict_w[w_key] = w_value - i = self.skip_whitespace(self.pos) + # this loop is a fast path for strings which do not contain escape + # characters ch = self.ll_chars[i] i += 1 - if ch == '}': + bits |= ord(ch) + if ch == '"': self.pos = i - return self._create_dict(dict_w) - elif ch == ',': - i = self.skip_whitespace(i) - else: - self._raise_object_error(ch, start, i - 1) + return self._create_string(start, i - 1, bits) + elif ch == '\\' or ch < '\x20': + self.pos = i-1 + return self.decode_string_escaped(start) - def decode_string_uncached(self, i): - start = i - ll_chars = self.ll_chars - nonascii, i = simd.find_end_of_string_no_hash(ll_chars, i, len(self.s)) - ch = ll_chars[i] - if ch == '\\': - self.pos = i - return self.decode_string_escaped(start, nonascii) - if ch < '\x20': - self._raise_control_char_in_string(ch, start, i) + def _create_string(self, start, end, bits): + if bits & 0x80: + # the 8th bit is set, it's an utf8 string + content_utf8 = self.getslice(start, end) + lgt = unicodehelper.check_utf8_or_raise(self.space, + content_utf8) + return self.space.newutf8(content_utf8, lgt) else: - assert ch == '"' - - self.pos = i + 1 - return self._create_string_wrapped(start, i, nonascii) - - def _create_string_wrapped(self, start, end, nonascii): - content = self.getslice(start, end) - if nonascii: - # contains non-ascii chars, we need to check that it's valid utf-8 - lgt = unicodehelper.check_utf8_or_raise(self.space, - content) - else: - lgt = end - start - return self.space.newutf8(content, lgt) + # ascii only, fast path (ascii is a strict subset of + # latin1, and we already checked that all the chars are < + # 128) + return self.space.newutf8(self.getslice(start, end), + end - start) def _create_dict(self, d): from pypy.objspace.std.dictmultiobject import from_unicode_key_dict @@ -446,7 +313,8 @@ from pypy.objspace.std.dictmultiobject import create_empty_unicode_key_dict return create_empty_unicode_key_dict(self.space) - def decode_string_escaped(self, start, nonascii): + + def decode_string_escaped(self, start): i = self.pos builder = StringBuilder((i - start) * 2) # just an estimate assert start >= 0 @@ -457,21 +325,25 @@ i += 1 if ch == '"': content_utf8 = builder.build() - length = unicodehelper.check_utf8_or_raise(self.space, + lgt = unicodehelper.check_utf8_or_raise(self.space, content_utf8) self.pos = i - return self.space.newutf8(content_utf8, length) + return self.space.newutf8(content_utf8, lgt) elif ch == '\\': - i = self.decode_escape_sequence_to_utf8(i, builder) + i = self.decode_escape_sequence(i, builder) elif ch < '\x20': - self._raise_control_char_in_string(ch, start, i) + if ch == '\0': + self._raise("Unterminated string starting at char %d", + start - 1) + else: + self._raise("Invalid control character at char %d", i-1) else: builder.append(ch) - def decode_escape_sequence_to_utf8(self, i, stringbuilder): + def decode_escape_sequence(self, i, builder): ch = self.ll_chars[i] i += 1 - put = stringbuilder.append + put = builder.append if ch == '\\': put('\\') elif ch == '"': put('"' ) elif ch == '/': put('/' ) @@ -481,37 +353,22 @@ elif ch == 'r': put('\r') elif ch == 't': put('\t') elif ch == 'u': - # may be a suggorate pair - return self.decode_escape_sequence_unicode(i, stringbuilder) + return self.decode_escape_sequence_unicode(i, builder) else: self._raise("Invalid \\escape: %s (char %d)", ch, i-1) return i - def _get_int_val_from_hex4(self, i): - ll_chars = self.ll_chars - res = 0 - for i in range(i, i + 4): - ch = ord(ll_chars[i]) - if ord('a') <= ch <= ord('f'): - digit = ch - ord('a') + 10 - elif ord('A') <= ch <= ord('F'): - digit = ch - ord('A') + 10 - elif ord('0') <= ch <= ord('9'): - digit = ch - ord('0') - else: - raise ValueError - res = (res << 4) + digit - return res - def decode_escape_sequence_unicode(self, i, builder): # at this point we are just after the 'u' of the \u1234 sequence. start = i i += 4 + hexdigits = self.getslice(start, i) try: - val = self._get_int_val_from_hex4(start) + val = int(hexdigits, 16) if (0xd800 <= val <= 0xdbff and self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u'): - lowsurr = self._get_int_val_from_hex4(i + 2) + hexdigits = self.getslice(i+2, i+6) + lowsurr = int(hexdigits, 16) if 0xdc00 <= lowsurr <= 0xdfff: # decode surrogate pair val = 0x10000 + (((val - 0xd800) << 10) | @@ -526,531 +383,45 @@ builder.append(utf8_ch) return i + def decode_key(self, i): + """ returns a wrapped unicode """ + from rpython.rlib.rarithmetic import intmask - def decode_string(self, i, context=None): - ll_chars = self.ll_chars - start = i - ch = ll_chars[i] - if ch == '"': - self.pos = i + 1 - return self.w_empty_string # surprisingly common - - cache = True - if context is not None: - context.decoded_strings += 1 - if not context.should_cache(): - cache = False - if len(self.s) < self.MIN_SIZE_FOR_STRING_CACHE: - cache = False - - if not cache: - return self.decode_string_uncached(i) - - strhash, nonascii, i = simd.find_end_of_string(ll_chars, i, len(self.s)) - ch = ll_chars[i] - if ch == '\\': - self.pos = i - return self.decode_string_escaped(start, nonascii) - if ch < '\x20': - self._raise_control_char_in_string(ch, start, i) - else: - assert ch == '"' - - self.pos = i + 1 - - length = i - start - strhash ^= length - - # check cache first: - try: - entry = self.cache_wrapped[strhash] - except KeyError: - w_res = self._create_string_wrapped(start, i, nonascii) - # only add *some* strings to the cache, because keeping them all is - # way too expensive - if (context is not None and context.decoded_strings < 200) or strhash in self.lru_cache: - entry = WrappedCacheEntry( - self.getslice(start, start + length), w_res) - self.cache_wrapped[strhash] = entry - else: - self.lru_cache[self.lru_index] = strhash - self.lru_index = (self.lru_index + 1) & self.LRU_MASK - return w_res - if not _compare_cache_entry(self.space, entry.repr, ll_chars, start, length): - # hopefully rare - return self._create_string_wrapped(start, i, nonascii) - if context is not None: - context.cache_hits += 1 - return entry.w_uni - - def decode_key(self, i, currmap): - newmap = self._decode_key(i, currmap) - currmap.observe_transition(newmap) - return newmap - - def _decode_key(self, i, currmap): - ll_chars = self.ll_chars - nextmap = currmap.fast_path_key_parse(self, i) - if nextmap is not None: - return nextmap - - start = i - ch = ll_chars[i] - if ch != '"': - self._raise("Key name must be string at char %d", i) - i += 1 - w_key = self._decode_key_string(i) - return currmap.get_next(w_key, self.s, start, self.pos) - - def _decode_key_string(self, i): - ll_chars = self.ll_chars - start = i - - strhash, nonascii, i = simd.find_end_of_string(ll_chars, i, len(self.s)) - - ch = ll_chars[i] - if ch == '\\': - self.pos = i - w_key = self.decode_string_escaped(start, nonascii) - return w_key - if ch < '\x20': - self._raise_control_char_in_string(ch, start, i) - length = i - start - strhash ^= length - self.pos = i + 1 - # check cache first: - try: - entry = self.cache[strhash] - except KeyError: - w_res = self._create_string_wrapped(start, i, nonascii) - entry = WrappedCacheEntry( - self.getslice(start, start + length), w_res) - self.cache[strhash] = entry - return w_res - if not _compare_cache_entry(self.space, entry.repr, ll_chars, start, length): - # hopefully rare - w_res = self._create_string_wrapped(start, i, nonascii) - print w_res - else: - w_res = entry.w_uni - return w_res - - def decode_key_string(self, i): + i = self.skip_whitespace(i) ll_chars = self.ll_chars ch = ll_chars[i] if ch != '"': self._raise("Key name must be string at char %d", i) i += 1 - return self._decode_key_string(i) -class WrappedCacheEntry(object): - def __init__(self, repr, w_uni): - self.repr = repr - self.w_uni = w_uni - - -class MapBase(object): - # the basic problem we are trying to solve is the following: dicts in - # json can either be used as objects, or as dictionaries with arbitrary - # string keys. We want to use maps for the former, but not for the - # latter. But we don't know in advance which kind of dict is which. - - # Therefore we create "preliminary" maps where we aren't quite sure yet - # whether they are really useful maps or not. If we see them used often - # enough, we promote them to "useful" maps, which we will actually - # instantiate objects with. - - # If we determine that a map is not used often enough, we can turn it - # into a "blocked" map, which is a point in the map tree where we will - # switch to regular dicts, when we reach that part of the tree. - - # allowed graph edges or nodes in all_next: - # USEFUL ------- - # / \ \ - # v v v - # FRINGE USEFUL BLOCKED - # | - # v - # PRELIMINARY - # | - # v - # PRELIMINARY - - # state transitions: - # PRELIMINARY - # / | \ - # | v v - # | FRINGE -> USEFUL - # | | - # \ | - # v v - # BLOCKED - - # the single_nextmap edge can only be these graph edges: - # USEFUL - # | - # v - # USEFUL - # - # FRINGE - # | - # v - # PRELIMINARY - # | - # v - # PRELIMINARY - - USEFUL = 'u' - PRELIMINARY = 'p' - FRINGE = 'f' # buffer between PRELIMINARY and USEFUL - BLOCKED = 'b' - - # tunable parameters - MAX_FRINGE = 40 - USEFUL_THRESHOLD = 5 - - def __init__(self, space): - self.space = space - - # a single transition is stored in .single_nextmap - self.single_nextmap = None - - # all_next is only initialized after seeing the *second* transition - # but then it also contains .single_nextmap - self.all_next = None # later dict {key: nextmap} - - self.instantiation_count = 0 - self.number_of_leaves = 1 - - def get_terminator(self): - while isinstance(self, JSONMap): - self = self.prev - assert isinstance(self, Terminator) - return self - - def _check_invariants(self): - if self.all_next: - for next in self.all_next.itervalues(): - next._check_invariants() - elif self.single_nextmap: - self.single_nextmap._check_invariants() - - def get_next(self, w_key, string, start, stop): - from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq - if isinstance(self, JSONMap): - assert not self.state == MapBase.BLOCKED - single_nextmap = self.single_nextmap - if (single_nextmap is not None and - single_nextmap.w_key.eq_w(w_key)): - return single_nextmap - - assert stop >= 0 - assert start >= 0 - - if single_nextmap is None: - # first transition ever seen, don't initialize all_next - next = self._make_next_map(w_key, string[start:stop]) - self.single_nextmap = next + start = i + bits = 0 + strhash = ord(ll_chars[i]) << 7 + while True: + ch = ll_chars[i] + i += 1 + if ch == '"': + break + elif ch == '\\' or ch < '\x20': + self.pos = i-1 + return self.decode_string_escaped(start) + strhash = intmask((1000003 * strhash) ^ ord(ll_chars[i])) + bits |= ord(ch) + length = i - start - 1 + if length == 0: + strhash = -1 else: - if self.all_next is None: - self.all_next = objectmodel.r_dict(unicode_eq, unicode_hash, - force_non_null=True, simple_hash_eq=True) - self.all_next[single_nextmap.w_key] = single_nextmap - else: - next = self.all_next.get(w_key, None) - if next is not None: - return next - next = self._make_next_map(w_key, string[start:stop]) - self.all_next[w_key] = next - - # fix number_of_leaves - self.change_number_of_leaves(1) - - terminator = self.get_terminator() - terminator.register_potential_fringe(next) - return next - - def change_number_of_leaves(self, difference): - parent = self - while isinstance(parent, JSONMap): - parent.number_of_leaves += difference - parent = parent.prev - parent.number_of_leaves += difference # terminator - - def fast_path_key_parse(self, decoder, position): - single_nextmap = self.single_nextmap - if single_nextmap: - ll_chars = decoder.ll_chars - assert isinstance(single_nextmap, JSONMap) - if single_nextmap.key_repr_cmp(ll_chars, position): - decoder.pos = position + len(single_nextmap.key_repr) - return single_nextmap - - def observe_transition(self, newmap): - """ observe a transition from self to newmap. - This does a few things, including updating the self size estimate with - the knowledge that one object transitioned from self to newmap. - also it potentially decides that self should move to state USEFUL.""" - self.instantiation_count += 1 - if isinstance(self, JSONMap) and self.state == MapBase.FRINGE: - if self.is_useful(): - self.mark_useful() - - def _make_next_map(self, w_key, key_repr): - return JSONMap(self.space, self, w_key, key_repr) - - def _all_dot(self, output): - identity = objectmodel.compute_unique_id(self) - output.append('%s [shape=box%s];' % (identity, self._get_dot_text())) - if self.all_next: - for w_key, value in self.all_next.items(): - assert isinstance(value, JSONMap) - if value is self.single_nextmap: - color = ", color=blue" - else: - color = "" - output.append('%s -> %s [label="%s"%s];' % ( - identity, objectmodel.compute_unique_id(value), value.w_key._utf8, color)) - value._all_dot(output) - elif self.single_nextmap is not None: - value = self.single_nextmap - output.append('%s -> %s [label="%s", color=blue];' % ( - identity, objectmodel.compute_unique_id(value), value.w_key._utf8)) - value._all_dot(output) - - - def _get_dot_text(self): - return ", label=base" - - def view(self): - from dotviewer import graphclient - import pytest - r = ["digraph G {"] - self._all_dot(r) - r.append("}") - p = pytest.ensuretemp("resilientast").join("temp.dot") - p.write("\n".join(r)) - graphclient.display_dot_file(str(p)) - - def _get_caching_stats(self): - caching = 0 - num_maps = 1 - if isinstance(self, JSONMap) and self.should_cache() and self.decoded_strings > 200: - caching += 1 - - if self.all_next: - children = self.all_next.values() - elif self.single_nextmap: - children = [self.single_nextmap] - else: - children = [] - for child in children: - a, b = child._get_caching_stats() - caching += a - num_maps += b - return caching, num_maps - -class Terminator(MapBase): - def __init__(self, space): - MapBase.__init__(self, space) - self.all_object_count = 0 - self.current_fringe = {} - - def register_potential_fringe(self, prelim): - prev = prelim.prev - if (isinstance(prev, Terminator) or - isinstance(prev, JSONMap) and prev.state == MapBase.USEFUL): - prelim.state = MapBase.FRINGE - - if len(self.current_fringe) > MapBase.MAX_FRINGE: - self.cleanup_fringe() - self.current_fringe[prelim] = None - - def cleanup_fringe(self): - min_fringe = None - min_avg = 10000000000 - for f in self.current_fringe: - if f.state == MapBase.FRINGE: - avg = f.average_instantiation() - if avg < min_avg: - min_avg = avg - min_fringe = f - else: - for f in self.current_fringe.keys(): - if f.state != MapBase.FRINGE: - del self.current_fringe[f] - return - assert min_fringe - min_fringe.mark_blocked() - del self.current_fringe[min_fringe] - - -class JSONMap(MapBase): - """ A map implementation to speed up parsing """ - - def __init__(self, space, prev, w_key, key_repr): - MapBase.__init__(self, space) - - self.prev = prev - self.w_key = w_key - self.key_repr = key_repr - - self.state = MapBase.PRELIMINARY - - # key decoding stats - self.decoded_strings = 0 - self.cache_hits = 0 - - # for jsondict support - self.key_to_index = None - self.keys_in_order = None - self.strategy_instance = None - - @jit.elidable - def get_terminator(self): - while isinstance(self, JSONMap): - self = self.prev - assert isinstance(self, Terminator) - return self - - def _check_invariants(self): - assert self.state in ( - MapBase.USEFUL, - MapBase.PRELIMINARY, - MapBase.FRINGE, - MapBase.BLOCKED, - ) - - prev = self.prev - if isinstance(prev, JSONMap): - prevstate = prev.state - else: - prevstate = MapBase.USEFUL - - if prevstate == MapBase.USEFUL: - assert self.state != MapBase.PRELIMINARY - elif prevstate == MapBase.PRELIMINARY: - assert self.state == MapBase.PRELIMINARY - elif prevstate == MapBase.FRINGE: - assert self.state == MapBase.PRELIMINARY - else: - # if prevstate is BLOCKED, we shouldn't have recursed here! - assert False, "should be unreachable" - - if self.state == MapBase.BLOCKED: - assert self.single_nextmap is None - assert self.all_next is None - - MapBase._check_invariants(self) - - def mark_useful(self): - # mark self as useful, and also the most commonly instantiated - # children, recursively - assert self.state in (MapBase.FRINGE, MapBase.PRELIMINARY) - self.state = MapBase.USEFUL - maxchild = self.single_nextmap - if self.all_next is not None: - for child in self.all_next.itervalues(): - if child.instantiation_count > maxchild.instantiation_count: - maxchild = child - if maxchild is not None: - maxchild.mark_useful() - if self.all_next: - terminator = self.get_terminator() - for child in self.all_next.itervalues(): - if child is not maxchild: - terminator.register_potential_fringe(child) - self.single_nextmap = maxchild - - def mark_blocked(self): - self.state = MapBase.BLOCKED - if self.all_next: - for next in self.all_next.itervalues(): - next.mark_blocked() - elif self.single_nextmap: - self.single_nextmap.mark_blocked() - self.single_nextmap = None - self.all_next = None - self.change_number_of_leaves(-self.number_of_leaves + 1) - - def is_blocked(self): - return self.state == MapBase.BLOCKED - - def average_instantiation(self): - return self.instantiation_count / float(self.number_of_leaves) - - def is_useful(self): - return self.average_instantiation() > self.USEFUL_THRESHOLD - - def should_cache(self): - return not (self.decoded_strings > 200 and self.cache_hits * 4 < self.decoded_strings) - - def key_repr_cmp(self, ll_chars, i): - for j, c in enumerate(self.key_repr): - if ll_chars[i] != c: - return False - i += 1 - return True - - # _____________________________________________________ - # methods for JsonDictStrategy - - @jit.elidable - def get_index(self, w_key): - from pypy.objspace.std.unicodeobject import W_UnicodeObject - assert isinstance(w_key, W_UnicodeObject) - return self.get_key_to_index().get(w_key, -1) - - def get_key_to_index(self): - from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq - key_to_index = self.key_to_index - if key_to_index is None: - key_to_index = self.key_to_index = objectmodel.r_dict(unicode_eq, unicode_hash, - force_non_null=True, simple_hash_eq=True) - # compute depth - curr = self - depth = 0 - while True: - depth += 1 - curr = curr.prev - if not isinstance(curr, JSONMap): - break - - curr = self - while depth: - depth -= 1 - key_to_index[curr.w_key] = depth - curr = curr.prev - if not isinstance(curr, JSONMap): - break - return key_to_index - - def get_keys_in_order(self): - keys_in_order = self.keys_in_order - if keys_in_order is None: - key_to_index = self.get_key_to_index() - keys_in_order = self.keys_in_order = [None] * len(key_to_index) - for w_key, index in key_to_index.iteritems(): - keys_in_order[index] = w_key - return keys_in_order - - # _____________________________________________________ - - def _get_dot_text(self): - if self.all_next is None: - l = int(self.single_nextmap is not None) - else: - l = len(self.all_next) + 1 - extra = "" - if self.decoded_strings: - extra = "\\n%s/%s (%s%%)" % (self.cache_hits, self.decoded_strings, self.cache_hits/float(self.decoded_strings)) - res = ', label="#%s\\nchildren: %s%s"' % (self.instantiation_count, l, extra) - if self.state == MapBase.BLOCKED: - res += ", fillcolor=lightsalmon" - if self.state == MapBase.FRINGE: - res += ", fillcolor=lightgray" - if self.state == MapBase.PRELIMINARY: - res += ", fillcolor=lightslategray" + strhash ^= length + strhash = intmask(strhash) + self.pos = i + # check cache first: + key = (ll_chars, start, length, strhash) + try: + return self.cache[key] + except KeyError: + pass + res = self._create_string(start, i - 1, bits) + self.cache[key] = res return res @@ -1071,4 +442,3 @@ return w_res finally: decoder.close() - diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py deleted file mode 100644 --- a/pypy/module/_pypyjson/simd.py +++ /dev/null @@ -1,218 +0,0 @@ -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib import objectmodel, unroll -from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT -from rpython.jit.backend.detect_cpu import autodetect - -# accelerators for string operations using simd on regular word sizes (*not* -# SSE instructions). this style is sometimes called SWAR (SIMD Within A -# Register) or "broadword techniques" - -# XXX remove wordsize and endianness restrictions properly, so far only x86-64 -# is tested - -USE_SIMD = False -if LONG_BIT == 64: - WORD_SIZE = 8 - EVERY_BYTE_ONE = 0x0101010101010101 - EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080 - if autodetect() == "x86-64": - USE_SIMD = True -else: - WORD_SIZE = 4 - EVERY_BYTE_ONE = 0x01010101 - EVERY_BYTE_HIGHEST_BIT = 0x80808080 - - -# helpers - -unrolling_wordsize = unroll.unrolling_iterable(range(WORD_SIZE)) - -def char_repeated_word_width(ch): - return r_uint(EVERY_BYTE_ONE) * ord(ch) - -def any_char_zero(word): - return (word - r_uint(EVERY_BYTE_ONE)) & ~word & r_uint(EVERY_BYTE_HIGHEST_BIT) - -def any_char_in_words_zero(*words): - return _any_char_in_any_word_zero_accum(0, *words) - -def _any_char_in_any_word_zero_accum(accum, word, *words): - accum |= (word - r_uint(EVERY_BYTE_ONE)) & ~word - if not words: - return accum & r_uint(EVERY_BYTE_HIGHEST_BIT) - return _any_char_in_any_word_zero_accum(accum, *words) - -def print_chars(word): - # for debugging - out = '' - for i in range(WORD_SIZE): - out += chr(word & 0xff) - word >>= 8 - return out - -def index_nonzero(word): - # XXX can be done very cheap in theory - assert word - for i in unrolling_wordsize: - if word & 0xff: - return i - word >>= 8 - assert 0 - -def index_zero(word): - # XXX can be done very cheap in theory - assert any_char_zero(word) - for i in unrolling_wordsize: - if not word & 0xff: - return i - word >>= 8 - assert 0 # XXX ??? - -def splice_words(word, offset, other): - mask = ((~r_uint(0)) << (8 * offset)) - return (word & mask) | (other & ~mask) - - - - at objectmodel.always_inline -def position_string_ender(word): - maskquote = char_repeated_word_width('"') - maskbackslash = char_repeated_word_width('\\') - maskx20 = char_repeated_word_width(chr(0xff - 0x1f)) - # x1 and x2 check for equality, if a byte is 0 the corresponding - # char is equal to " or \ - x1 = maskquote ^ word - x2 = maskbackslash ^ word - # x3 checks for char < 0x20, the byte is 0 in that case - x3 = maskx20 & word - return any_char_in_words_zero(x1, x2, x3) - - at objectmodel.always_inline -def find_end_of_string_simd_unaligned(ll_chars, startpos, length): - ch = ll_chars[startpos] - strhash = (ord(ch) << 7) ^ 0x345678 - - wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) - num_safe_reads = (length - startpos) // WORD_SIZE - - bits = 0 - for i in range(num_safe_reads): - word = wordarray[i] - cond = position_string_ender(word) - if cond: - break - bits |= word - strhash = intmask((1000003 * strhash) ^ intmask(word)) - else: - # didn't find end of string yet, look at remaining chars - word = 0 - shift = 0 - i = 0 - for i in range(num_safe_reads * WORD_SIZE + startpos, length + 1): - ch = ll_chars[i] - if ch == '"' or ch == '\\' or ch < '\x20': - break - bits |= ord(ch) - word |= ord(ch) << shift - shift += 8 - if shift: - strhash = intmask((1000003 * strhash) ^ intmask(word)) - - nonascii = bool(bits & char_repeated_word_width(chr(0x80))) - return strhash, nonascii, i - - # compute endposition - nonzero = index_nonzero(cond) - endposition = startpos + i * WORD_SIZE + nonzero - if nonzero: - word = splice_words(r_uint(0), nonzero, word) - bits |= word - strhash = intmask((1000003 * strhash) ^ intmask(word)) - - nonascii = bool(bits & char_repeated_word_width(chr(0x80))) - - return strhash, nonascii, endposition - - at objectmodel.always_inline -def find_end_of_string_simd_unaligned_no_hash(ll_chars, startpos, length): - ch = ll_chars[startpos] - - wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) - num_safe_reads = (length - startpos) // WORD_SIZE - - bits = 0 - for i in range(num_safe_reads): - word = wordarray[i] - cond = position_string_ender(word) - if cond: - break - bits |= word - else: - # didn't find end of string yet, look at remaining chars - word = 0 - shift = 0 - i = 0 - for i in range(num_safe_reads * WORD_SIZE + startpos, length + 1): - ch = ll_chars[i] - if ch == '"' or ch == '\\' or ch < '\x20': - break - bits |= ord(ch) - word |= ord(ch) << shift - shift += WORD_SIZE - - nonascii = bool(bits & char_repeated_word_width(chr(0x80))) - return nonascii, i - - # compute endposition - nonzero = index_nonzero(cond) - endposition = startpos + i * WORD_SIZE + nonzero - if nonzero: - word = splice_words(r_uint(0), nonzero, word) - bits |= word - - nonascii = bool(bits & char_repeated_word_width(chr(0x80))) - - return nonascii, endposition - - - at objectmodel.always_inline -def find_end_of_string_slow(ll_chars, i, length): - ch = ll_chars[i] - strhash = (ord(ch) << 7) ^ 0x345678 - word = 0 - shift = 0 - - bits = 0 - - while True: - # this loop is a fast path for strings which do not contain escape - # characters - ch = ll_chars[i] - if ch == '"' or ch == '\\' or ch < '\x20': - break - i += 1 - bits |= ord(ch) - - word |= ord(ch) << shift - shift += WORD_SIZE - if shift == WORD_SIZE * WORD_SIZE: - strhash = intmask((1000003 * strhash) ^ word) - shift = 0 - word = 0 - - if shift: - strhash = intmask((1000003 * strhash) ^ word) - return strhash, bool(bits & 0x80), i - -if USE_SIMD: - find_end_of_string = find_end_of_string_simd_unaligned - find_end_of_string_no_hash = find_end_of_string_simd_unaligned_no_hash -else: - find_end_of_string = find_end_of_string_slow - - @objectmodel.always_inline - def find_end_of_string_no_hash(ll_chars, i, length): - _, nonascii, i = find_end_of_string_slow(ll_chars, i, length) - return (nonascii, i) - - diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -1,253 +1,31 @@ # -*- encoding: utf-8 -*- -import pytest -from pypy.module._pypyjson.interp_decoder import JSONDecoder, Terminator, MapBase -from rpython.rtyper.lltypesystem import lltype, rffi +from pypy.module._pypyjson.interp_decoder import JSONDecoder +def test_skip_whitespace(): + s = ' hello ' + dec = JSONDecoder('fake space', s) + assert dec.pos == 0 + assert dec.skip_whitespace(0) == 3 + assert dec.skip_whitespace(3) == 3 + assert dec.skip_whitespace(8) == len(s) + dec.close() -class TestJson(object): - def test_skip_whitespace(self): - s = ' hello ' - dec = JSONDecoder(self.space, s) - assert dec.pos == 0 - assert dec.skip_whitespace(0) == 3 - assert dec.skip_whitespace(3) == 3 - assert dec.skip_whitespace(8) == len(s) - dec.close() +class FakeSpace(object): + def newutf8(self, s, l): + return s - def test_json_map(self): - m = Terminator(self.space) - w_a = self.space.newutf8("a", 1) - w_b = self.space.newutf8("b", 1) - w_c = self.space.newutf8("c", 1) - m1 = m.get_next(w_a, '"a"', 0, 3) - assert m1.w_key == w_a - assert m1.single_nextmap is None - assert m1.key_repr == '"a"' - assert m1.key_repr_cmp('"a": 123', 0) - assert not m1.key_repr_cmp('b": 123', 0) - assert m.single_nextmap.w_key == w_a - - m2 = m.get_next(w_a, '"a"', 0, 3) - assert m2 is m1 - - m3 = m.get_next(w_b, '"b"', 0, 3) - assert m3.w_key == w_b - assert m3.single_nextmap is None - assert m3.key_repr == '"b"' - assert m.single_nextmap is m1 - - m4 = m3.get_next(w_c, '"c"', 0, 3) - assert m4.w_key == w_c - assert m4.single_nextmap is None - assert m4.key_repr == '"c"' - assert m3.single_nextmap is m4 - - def test_json_map_get_index(self): - m = Terminator(self.space) - w_a = self.space.newutf8("a", 1) - w_b = self.space.newutf8("b", 1) - w_c = self.space.newutf8("c", 1) - m1 = m.get_next(w_a, 'a"', 0, 2) - assert m1.get_index(w_a) == 0 - assert m1.get_index(w_b) == -1 - - m2 = m.get_next(w_b, 'b"', 0, 2) - assert m2.get_index(w_b) == 0 - assert m2.get_index(w_a) == -1 - - m3 = m2.get_next(w_c, 'c"', 0, 2) - assert m3.get_index(w_b) == 0 - assert m3.get_index(w_c) == 1 - assert m3.get_index(w_a) == -1 - - def test_decode_key(self): - m = Terminator(self.space) - m_diff = Terminator(self.space) - for s1 in ["abc", "1001" * 10, u"ä".encode("utf-8")]: - s = ' "%s" "%s" "%s"' % (s1, s1, s1) - dec = JSONDecoder(self.space, s) - assert dec.pos == 0 - m1 = dec.decode_key(dec.skip_whitespace(0), m) - assert m1.w_key._utf8 == s1 - assert m1.key_repr == '"%s"' % s1 - - # check caching on w_key level - m2 = dec.decode_key(dec.skip_whitespace(dec.pos), m_diff) - assert m1.w_key is m2.w_key - - # check caching on map level - m3 = dec.decode_key(dec.skip_whitespace(dec.pos), m_diff) - assert m3 is m2 - dec.close() - - def test_decode_string_caching(self): - for s1 in ["abc", u"ä".encode("utf-8")]: - s = '"%s" "%s" "%s"' % (s1, s1, s1) - dec = JSONDecoder(self.space, s) - dec.MIN_SIZE_FOR_STRING_CACHE = 0 - assert dec.pos == 0 - w_x = dec.decode_string(1) - w_y = dec.decode_string(dec.skip_whitespace(dec.pos) + 1) - assert w_x is not w_y - # check caching - w_z = dec.decode_string(dec.skip_whitespace(dec.pos) + 1) - assert w_z is w_y - dec.close() - - def _make_some_maps(self): - # base -> m1 -> m2 -> m3 - # \-> m4 - w_a = self.space.newutf8("a", 1) - w_b = self.space.newutf8("b", 1) - w_c = self.space.newutf8("c", 1) - w_d = self.space.newutf8("d", 1) - base = Terminator(self.space) - base.instantiation_count = 6 - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = m1.get_next(w_b, 'b"', 0, 2) - m3 = m2.get_next(w_c, 'c"', 0, 2) - m4 = m2.get_next(w_d, 'd"', 0, 2) - return base, m1, m2, m3, m4 - - # unit tests for map state transistions - def test_fringe_to_useful(self): - base, m1, m2, m3, m4 = self._make_some_maps() - base.instantiation_count = 6 - assert m1.state == MapBase.FRINGE - m1.instantiation_count = 6 - - assert m2.state == MapBase.PRELIMINARY - m2.instantiation_count = 6 - - assert m3.state == MapBase.PRELIMINARY - m3.instantiation_count = 2 - assert m2.single_nextmap is m3 - - assert m4.state == MapBase.PRELIMINARY - m4.instantiation_count = 4 - - m1.mark_useful() - assert m1.state == MapBase.USEFUL - assert m2.state == MapBase.USEFUL - assert m3.state == MapBase.FRINGE - assert m4.state == MapBase.USEFUL - assert m2.single_nextmap is m4 - - assert m1.number_of_leaves == 2 - base._check_invariants() - - def test_number_of_leaves(self): - w_x = self.space.newutf8("x", 1) - base, m1, m2, m3, m4 = self._make_some_maps() - assert base.number_of_leaves == 2 - assert m1.number_of_leaves == 2 - assert m2.number_of_leaves == 2 - assert m3.number_of_leaves == 1 - assert m4.number_of_leaves == 1 - m5 = m2.get_next(w_x, 'x"', 0, 2) - assert base.number_of_leaves == 3 - assert m1.number_of_leaves == 3 - assert m2.number_of_leaves == 3 - assert m5.number_of_leaves == 1 - - def test_cleanup_fringe_simple(self): - base, m1, m2, m3, m4 = self._make_some_maps() - base.instantiation_count = 6 - assert m1.state == MapBase.FRINGE - m1.instantiation_count = 6 - m2.instantiation_count = 6 - m3.instantiation_count = 2 - m4.instantiation_count = 4 - assert base.current_fringe == {m1: None} - - m1.mark_useful() - assert base.current_fringe == {m1: None, m3: None} # not cleaned up - base.cleanup_fringe() - assert base.current_fringe == {m3: None} - - def test_cleanup_fringe_block(self): - w_a = self.space.newutf8("a", 1) - w_b = self.space.newutf8("b", 1) - w_c = self.space.newutf8("c", 1) - w_d = self.space.newutf8("d", 1) - base = Terminator(self.space) - base.instantiation_count = 6 - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = base.get_next(w_b, 'b"', 0, 2) - m3 = base.get_next(w_c, 'c"', 0, 2) - m4 = base.get_next(w_d, 'd"', 0, 2) - m5 = m4.get_next(w_a, 'a"', 0, 2) - base.instantiation_count = 7 - m1.instantiation_count = 2 - m2.instantiation_count = 2 - m3.instantiation_count = 2 - m4.instantiation_count = 1 - m5.instantiation_count = 1 - assert base.current_fringe == dict.fromkeys([m1, m2, m3, m4]) - - base.cleanup_fringe() - assert base.current_fringe == dict.fromkeys([m1, m2, m3]) - assert m4.state == MapBase.BLOCKED - assert m4.single_nextmap is None - assert m4.all_next is None - assert m5.state == MapBase.BLOCKED - assert m5.single_nextmap is None - assert m5.all_next is None - - def test_deal_with_blocked(self): - w_a = self.space.newutf8("a", 1) - w_b = self.space.newutf8("b", 1) - w_c = self.space.newutf8("c", 1) - space = self.space - s = '{"a": 1, "b": 2, "c": 3}' - dec = JSONDecoder(space, s) - dec.startmap = base = Terminator(space) - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = m1.get_next(w_b, 'b"', 0, 2) - m2.mark_blocked() - w_res = dec.decode_object(1) - assert space.int_w(space.len(w_res)) == 3 - assert space.int_w(space.getitem(w_res, w_a)) == 1 - assert space.int_w(space.getitem(w_res, w_b)) == 2 - assert space.int_w(space.getitem(w_res, w_c)) == 3 - dec.close() - - def test_deal_with_blocked_number_of_leaves(self): - w_a = self.space.newutf8("a", 1) - w_b = self.space.newutf8("b", 1) - w_x = self.space.newutf8("x", 1) - w_u = self.space.newutf8("u", 1) - space = self.space - base = Terminator(space) - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = m1.get_next(w_b, 'b"', 0, 2) - m2.get_next(w_x, 'x"', 0, 2) - m2.get_next(w_u, 'u"', 0, 2) - assert base.number_of_leaves == 2 - m2.mark_blocked() - assert base.number_of_leaves == 1 - - @pytest.mark.skip() - def test_caching_stats(self): - w_a = self.space.newutf8("a", 1) - w_b = self.space.newutf8("b", 1) - w_x = self.space.newutf8("x", 1) - w_u = self.space.newutf8("u", 1) - space = self.space - base = Terminator(space) - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = m1.get_next(w_b, 'b"', 0, 2) - m2.get_next(w_x, 'x"', 0, 2) - m2.get_next(w_u, 'u"', 0, 2) - m1.decode_string = 300 - m1.cache_hits = 0 - m3 = base.get_next(w_b, '"b"', 0, 3) - m3.decode_string = 300 - m3.cache_hits = 300 - caching_maps, total_maps = base._get_caching_stats() - assert caching_maps == 5 - assert total_maps == 6 - +def test_decode_key(): + s1 = "123" * 100 + s = ' "%s" "%s" ' % (s1, s1) + dec = JSONDecoder(FakeSpace(), s) + assert dec.pos == 0 + x = dec.decode_key(0) + assert x == s1 + # check caching + y = dec.decode_key(dec.pos) + assert y == s1 + assert y is x + dec.close() class AppTest(object): spaceconfig = {"objspace.usemodules._pypyjson": True} @@ -277,7 +55,7 @@ raises(ValueError, _pypyjson.loads, 'fa') raises(ValueError, _pypyjson.loads, 'f') raises(ValueError, _pypyjson.loads, 'falXX') - + def test_decode_string(self): import _pypyjson @@ -307,7 +85,7 @@ import _pypyjson assert _pypyjson.loads(r'"\\"') == u'\\' assert _pypyjson.loads(r'"\""') == u'"' - assert _pypyjson.loads(r'"\/"') == u'/' + assert _pypyjson.loads(r'"\/"') == u'/' assert _pypyjson.loads(r'"\b"') == u'\b' assert _pypyjson.loads(r'"\f"') == u'\f' assert _pypyjson.loads(r'"\n"') == u'\n' @@ -323,19 +101,12 @@ import _pypyjson s = r'"hello\nworld' # missing the trailing " raises(ValueError, "_pypyjson.loads(s)") - + def test_escape_sequence_unicode(self): import _pypyjson s = r'"\u1234"' assert _pypyjson.loads(s) == u'\u1234' - def test_escape_sequence_mixed_with_utf8(self): - import _pypyjson - utf8 = u'ä"'.encode("utf-8") - assert _pypyjson.loads(r'"abc\\' + utf8) == u'abc\\ä' - assert _pypyjson.loads(r'"abc\"' + utf8) == u'abc"ä' - assert _pypyjson.loads(r'"def\u1234' + utf8) == u'def\u1234ä' - def test_invalid_utf_8(self): import _pypyjson s = '"\xe0"' # this is an invalid UTF8 sequence inside a string @@ -405,18 +176,13 @@ s = '{"hello": "world", "aaa": "bbb"}' assert _pypyjson.loads(s) == {'hello': 'world', 'aaa': 'bbb'} - assert _pypyjson.loads(s) == {'hello': 'world', - 'aaa': 'bbb'} raises(ValueError, _pypyjson.loads, '{"key"') raises(ValueError, _pypyjson.loads, '{"key": 42') - assert _pypyjson.loads('{"neighborhood": ""}') == { - "neighborhood": ""} - def test_decode_object_nonstring_key(self): import _pypyjson raises(ValueError, "_pypyjson.loads('{42: 43}')") - + def test_decode_array(self): import _pypyjson assert _pypyjson.loads('[]') == [] @@ -497,4 +263,3 @@ for inputtext, errmsg in test_cases: exc = raises(ValueError, _pypyjson.loads, inputtext) assert str(exc.value) == errmsg - diff --git a/pypy/module/_pypyjson/test/test_simd.py b/pypy/module/_pypyjson/test/test_simd.py deleted file mode 100644 --- a/pypy/module/_pypyjson/test/test_simd.py +++ /dev/null @@ -1,104 +0,0 @@ -import sys -import pytest -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.rarithmetic import r_uint, intmask - -from pypy.module._pypyjson.simd import USE_SIMD -from pypy.module._pypyjson.simd import find_end_of_string_slow -from pypy.module._pypyjson.simd import print_chars -from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned, WORD_SIZE -from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned_no_hash - -from hypothesis import example, given, strategies - -if not USE_SIMD: - pytest.skip("only implemented for 64 bit for now") - -def fill_to_word_size(res, ch=" "): - if len(res) % WORD_SIZE != 0: - res += ch * (WORD_SIZE - (len(res) % WORD_SIZE)) - return res - -def string_to_word(s): - assert len(s) == WORD_SIZE - ll_chars, flag = rffi.get_nonmovingbuffer_final_null(s) - try: - wordarray = rffi.cast(rffi.ULONGP, ll_chars) - return wordarray[0] - finally: - rffi.free_nonmovingbuffer(s, ll_chars, flag) - -def ll(callable, string, *args): - ll_chars, flag = rffi.get_nonmovingbuffer_final_null(string) - try: - return callable(ll_chars, *args) - finally: - rffi.free_nonmovingbuffer(string, ll_chars, flag) - -word = strategies.builds( - r_uint, strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint)) - -def build_string(prefix, content, end, suffix): - res = prefix + '"' + "".join([chr(x) for x in content]) + end + suffix - return fill_to_word_size(res), len(prefix) + 1 - -string_in_context_strategy = strategies.builds( - build_string, prefix=strategies.binary(), - content=strategies.lists(strategies.integers(1, 255), min_size=1), - end=strategies.sampled_from('"\\\x00\x01'), - suffix=strategies.binary()) - -def compare(string, res1, res2): - hash1, nonascii1, endindex1 = res1 - hash2, nonascii2, endindex2 = res2 - assert endindex1 == endindex2 - if string[endindex1 - 1] == '"': - assert hash1 == hash2 - assert nonascii1 == nonascii2 - - - at example(('" \x80" ', 1)) - at example(('"\x01" ', 1)) - at example(('"aaaaaaaa"\x00\x00\x00\x00\x00\x00\x00 ', 1)) - at example(('"aaaaaaaa" ', 1)) - at example(('"12"', 1)) - at example(('"1234567abcdefghAB"', 1)) - at example(('"1234567abcdefgh"', 1)) - at example((' "123456ABCDEF" \x00', 2)) - at example((' "123456aaaaaaaaABCDEF"\x00', 2)) - at given(string_in_context_strategy) -def test_find_end_of_string(a): - (string, startindex) = a - res = ll(find_end_of_string_slow, string, startindex, len(string)) - hash, nonascii1, endposition1 = res - ch = string[endposition1] - assert ch == '"' or ch == '\\' or ch < '\x20' - for ch in string[startindex:endposition1]: - assert not (ch == '"' or ch == '\\' or ch < '\x20') - compare(string, res, ll(find_end_of_string_simd_unaligned, string, startindex, len(string))) - - nonascii2, endposition2 = ll(find_end_of_string_simd_unaligned_no_hash, string, startindex, len(string)) - assert nonascii1 == nonascii2 - assert endposition1 == endposition2 - - at given(string_in_context_strategy, strategies.binary(min_size=1)) -def test_find_end_of_string_position_invariance(a, prefix): - fn = find_end_of_string_simd_unaligned - (string, startindex) = a - h1, nonascii1, i1 = ll(fn, string, startindex, len(string)) - string2 = prefix + string - h2, nonascii2, i2 = ll(fn, string2, startindex + len(prefix), len(string) + len(prefix)) - assert h1 == h2 - assert nonascii1 == nonascii2 - assert i1 + len(prefix) == i2 - - at given(string_in_context_strategy, strategies.binary(min_size=1)) -def test_find_end_of_string_position_invariance_no_hash(a, prefix): - fn = find_end_of_string_simd_unaligned_no_hash - (string, startindex) = a - nonascii1, i1 = ll(fn, string, startindex, len(string)) - string2 = prefix + string - nonascii2, i2 = ll(fn, string2, startindex + len(prefix), len(string) + len(prefix)) - assert nonascii1 == nonascii2 - assert i1 + len(prefix) == i2 - diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py deleted file mode 100644 --- a/pypy/objspace/std/jsondict.py +++ /dev/null @@ -1,167 +0,0 @@ -"""dict implementation specialized for object loaded by the _pypyjson module. - -Somewhat similar to MapDictStrategy, also uses a map. -""" - -from rpython.rlib import jit, rerased, objectmodel, debug - -from pypy.objspace.std.dictmultiobject import ( - UnicodeDictStrategy, DictStrategy, - create_iterator_classes, W_DictObject) - - -def from_values_and_jsonmap(space, values_w, jsonmap): - if not objectmodel.we_are_translated(): - assert len(values_w) == len(jsonmap.get_keys_in_order()) - assert len(values_w) != 0 - debug.make_sure_not_resized(values_w) - strategy = jsonmap.strategy_instance - if strategy is None: - jsonmap.strategy_instance = strategy = JsonDictStrategy(space, jsonmap) - storage = strategy.erase(values_w) - return W_DictObject(space, strategy, storage) - -def devolve_jsonmap_dict(w_dict): - assert isinstance(w_dict, W_DictObject) - strategy = w_dict.get_strategy() - assert isinstance(strategy, JsonDictStrategy) - strategy.switch_to_unicode_strategy(w_dict) - -def get_jsonmap_from_dict(w_dict): - assert isinstance(w_dict, W_DictObject) - strategy = w_dict.get_strategy() - assert isinstance(strategy, JsonDictStrategy) - return strategy.jsonmap - -class JsonDictStrategy(DictStrategy): - erase, unerase = rerased.new_erasing_pair("jsondict") - erase = staticmethod(erase) - unerase = staticmethod(unerase) - - _immutable_fields_ = ['jsonmap'] - - def __init__(self, space, jsonmap): - DictStrategy.__init__(self, space) - self.jsonmap = jsonmap - - def wrap(self, w_key): - return w_key - - def wrapkey(space, key): - return key - - def get_empty_storage(self): - raise NotImplementedError("should not be reachable") - - def is_correct_type(self, w_obj): - space = self.space - return space.is_w(space.type(w_obj), space.w_unicode) - - def _never_equal_to(self, w_lookup_type): - return False - - def length(self, w_dict): - return len(self.unerase(w_dict.dstorage)) - - def getitem(self, w_dict, w_key): - if self.is_correct_type(w_key): - return self.getitem_unicode(w_dict, w_key) - else: - self.switch_to_unicode_strategy(w_dict) - return w_dict.getitem(w_key) - - def getitem_unicode(self, w_dict, w_key): - storage_w = self.unerase(w_dict.dstorage) - if jit.isconstant(w_key): - jit.promote(self) - index = self.jsonmap.get_index(w_key) - if index == -1: - return None - return storage_w[index] - - def setitem(self, w_dict, w_key, w_value): - if self.is_correct_type(w_key): - storage_w = self.unerase(w_dict.dstorage) - index = self.jsonmap.get_index(w_key) - if index != -1: - storage_w[index] = w_value - return - self.switch_to_unicode_strategy(w_dict) - w_dict.setitem(w_key, w_value) - - def setdefault(self, w_dict, w_key, w_default): - if self.is_correct_type(w_key): - w_result = self.getitem_unicode(w_dict, w_key) - if w_result is not None: - return w_result - self.switch_to_unicode_strategy(w_dict) - return w_dict.setdefault(w_key, w_default) - - def delitem(self, w_dict, w_key): - self.switch_to_unicode_strategy(w_dict) - return w_dict.delitem(w_key) - - def popitem(self, w_dict): - self.switch_to_unicode_strategy(w_dict) - return w_dict.popitem() - - def switch_to_unicode_strategy(self, w_dict): - strategy = self.space.fromcache(UnicodeDictStrategy) - values_w = self.unerase(w_dict.dstorage) - storage = strategy.get_empty_storage() - d_new = strategy.unerase(storage) - keys_in_order = self.jsonmap.get_keys_in_order() - assert len(keys_in_order) == len(values_w) - for index, w_key in enumerate(keys_in_order): - assert w_key is not None - assert type(w_key) is self.space.UnicodeObjectCls - d_new[w_key] = values_w[index] - w_dict.set_strategy(strategy) - w_dict.dstorage = storage - - def w_keys(self, w_dict): - return self.space.newlist(self.jsonmap.get_keys_in_order()) - - def values(self, w_dict): - return self.unerase(w_dict.dstorage)[:] # to make resizable - - def items(self, w_dict): - space = self.space - storage_w = self.unerase(w_dict.dstorage) - res = [None] * len(storage_w) - for index, w_key in enumerate(self.jsonmap.get_keys_in_order()): - res[index] = space.newtuple([w_key, storage_w[index]]) - return res - - def getiterkeys(self, w_dict): - return iter(self.jsonmap.get_keys_in_order()) - - def getitervalues(self, w_dict): - storage_w = self.unerase(w_dict.dstorage) - return iter(storage_w) - - def getiteritems_with_hash(self, w_dict): - storage_w = self.unerase(w_dict.dstorage) - return ZipItemsWithHash(self.jsonmap.get_keys_in_order(), storage_w) - - -class ZipItemsWithHash(object): - def __init__(self, list1, list2): - assert len(list1) == len(list2) - self.list1 = list1 - self.list2 = list2 - self.i = 0 - - def __iter__(self): - return self - - def next(self): - i = self.i - if i >= len(self.list1): - raise StopIteration - self.i = i + 1 - w_key = self.list1[i] - return (w_key, self.list2[i], w_key.hash_w()) - - -create_iterator_classes(JsonDictStrategy) diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py deleted file mode 100644 --- a/pypy/objspace/std/test/test_jsondict.py +++ /dev/null @@ -1,89 +0,0 @@ - -class AppTest(object): - spaceconfig = {"objspace.usemodules._pypyjson": True} - - def test_check_strategy(self): - import __pypy__ - import _pypyjson - - d = _pypyjson.loads('{"a": 1}') - assert __pypy__.strategy(d) == "JsonDictStrategy" - d = _pypyjson.loads('{}') - assert __pypy__.strategy(d) == "EmptyDictStrategy" - - def test_simple(self): - import __pypy__ - import _pypyjson - - d = _pypyjson.loads('{"a": 1, "b": "x"}') - assert len(d) == 2 - assert d[u"a"] == 1 - assert d[u"b"] == u"x" - assert u"c" not in d - - d[u"a"] = 5 - assert d[u"a"] == 5 - assert __pypy__.strategy(d) == "JsonDictStrategy" - - # devolve it - assert not 1 in d - assert __pypy__.strategy(d) == "UnicodeDictStrategy" - assert len(d) == 2 - assert d[u"a"] == 5 - assert d[u"b"] == u"x" - assert u"c" not in d - - def test_setdefault(self): - import __pypy__ - import _pypyjson - - d = _pypyjson.loads('{"a": 1, "b": "x"}') - assert d.setdefault(u"a", "blub") == 1 - d.setdefault(u"x", 23) - assert __pypy__.strategy(d) == "UnicodeDictStrategy" - assert len(d) == 3 - assert d == {u"a": 1, u"b": "x", u"x": 23} - - def test_delitem(self): - import __pypy__ - import _pypyjson - - d = _pypyjson.loads('{"a": 1, "b": "x"}') - del d[u"a"] - assert __pypy__.strategy(d) == "UnicodeDictStrategy" - assert len(d) == 1 - assert d == {u"b": "x"} - - def test_popitem(self): - import __pypy__ - import _pypyjson - - d = _pypyjson.loads('{"a": 1, "b": "x"}') - k, v = d.popitem() - assert __pypy__.strategy(d) == "UnicodeDictStrategy" - if k == u"a": - assert v == 1 - assert len(d) == 1 - assert d == {u"b": "x"} - else: - assert v == u"x" - assert len(d) == 1 - assert d == {u"a": 1} - - def test_keys_value_items(self): - import __pypy__ - import _pypyjson - - d = _pypyjson.loads('{"a": 1, "b": "x"}') - assert d.keys() == [u"a", u"b"] - assert d.values() == [1, u"x"] - assert d.items() == [(u"a", 1), (u"b", u"x")] - - def test_iter_keys_value_items(self): - import __pypy__ - import _pypyjson - - d = _pypyjson.loads('{"a": 1, "b": "x"}') - assert list(d.iterkeys()) == [u"a", u"b"] - assert list(d.itervalues()) == [1, u"x"] - assert list(d.iteritems()) == [(u"a", 1), (u"b", u"x")] From pypy.commits at gmail.com Wed Jun 5 08:56:16 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 05:56:16 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: various improvements to the json decoder, particularly for big JSON files: Message-ID: <5cf7bbf0.1c69fb81.3a7f8.7472@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96756:eb54897d4861 Date: 2019-06-05 14:54 +0200 http://bitbucket.org/pypy/pypy/changeset/eb54897d4861/ Log: various improvements to the json decoder, particularly for big JSON files: - use maps for those objects that are instance-like (some heuristics to determine which are and which aren't) - maps use speculative parsing to try to parse the next-common key with strcmp - use a string cache to cache commonly reappearing strings, but only for instance fields where that is useful - use an int cache for small integers - use SIMD to find the end of strings efficiently still needs a bit of cleanup, but slowly getting there diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -1,11 +1,13 @@ import sys from rpython.rlib.rstring import StringBuilder -from rpython.rlib.objectmodel import specialize, always_inline, r_dict -from rpython.rlib import rfloat, rutf8 +from rpython.rlib.objectmodel import specialize, always_inline +from rpython.rlib import rfloat, runicode, jit, objectmodel, rutf8 from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rarithmetic import r_uint from pypy.interpreter.error import oefmt from pypy.interpreter import unicodehelper +from pypy.interpreter.baseobjspace import W_Root +from pypy.module._pypyjson import simd OVF_DIGITS = len(str(sys.maxint)) @@ -15,45 +17,93 @@ # precomputing negative powers of 10 is MUCH faster than using e.g. math.pow # at runtime NEG_POW_10 = [10.0**-i for i in range(16)] +del i + def neg_pow_10(x, exp): if exp >= len(NEG_POW_10): return 0.0 return x * NEG_POW_10[exp] -def slice_eq(a, b): - (ll_chars1, start1, length1, _) = a - (ll_chars2, start2, length2, _) = b - if length1 != length2: +def _compare_cache_entry(space, res, ll_chars, start, length): + if length != len(res): return False - j = start2 - for i in range(start1, start1 + length1): - if ll_chars1[i] != ll_chars2[j]: + index = start + for c in res: + x = ord(c) + if not ll_chars[index] == chr(x): return False - j += 1 + index += 1 return True -def slice_hash(a): - (ll_chars, start, length, h) = a - return h -TYPE_UNKNOWN = 0 -TYPE_STRING = 1 -class JSONDecoder(object): +class IntCache(object): + START = -10 + END = 256 + + def __init__(self, space): + self.space = space + self.cache = [self.space.newint(i) + for i in range(self.START, self.END)] + + def newint(self, intval): + if self.START <= intval < self.END: + return self.cache[intval - self.START] + return self.space.newint(intval) + + +class JSONDecoder(W_Root): + + LRU_SIZE = 16 + LRU_MASK = LRU_SIZE - 1 + + DEFAULT_SIZE_SCRATCH = 20 + + MIN_SIZE_FOR_STRING_CACHE = 1024 * 1024 + + # evaluate the string cache for 200 strings, before looking at the hit rate + # and deciding whether to keep doing it + STRING_CACHE_EVALUATION_SIZE = 200 + + # keep using the string cache if at least 25% of all decoded strings are a + # hit in the cache + STRING_CACHE_USEFULNESS_FACTOR = 4 + + def __init__(self, space, s): self.space = space + self.w_empty_string = space.newutf8("", 0) + self.s = s + # we put our string in a raw buffer so: # 1) we automatically get the '\0' sentinel at the end of the string, # which means that we never have to check for the "end of string" # 2) we can pass the buffer directly to strtod - self.ll_chars = rffi.str2charp(s) + self.ll_chars, self.flag = rffi.get_nonmovingbuffer_final_null(self.s) self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') self.pos = 0 - self.cache = r_dict(slice_eq, slice_hash, simple_hash_eq=True) + self.intcache = space.fromcache(IntCache) + + self.cache = {} + self.cache_wrapped = {} + + self.lru_cache = [0] * self.LRU_SIZE + self.lru_index = 0 + + self.startmap = self.space.fromcache(Terminator) + self.unclear_objects = [] + + self.scratch = [[None] * self.DEFAULT_SIZE_SCRATCH] # list of scratch space + def close(self): - rffi.free_charp(self.ll_chars) + rffi.free_nonmovingbuffer(self.s, self.ll_chars, self.flag) lltype.free(self.end_ptr, flavor='raw') + # clean up objects that are instances of now blocked maps + for w_obj in self.unclear_objects: + jsonmap = self._get_jsonmap_from_dict(w_obj) + if jsonmap.is_blocked(): + self._devolve_jsonmap_dict(w_obj) def getslice(self, start, end): assert start >= 0 @@ -61,18 +111,15 @@ return self.s[start:end] def skip_whitespace(self, i): + ll_chars = self.ll_chars while True: - ch = self.ll_chars[i] + ch = ll_chars[i] if is_whitespace(ch): - i+=1 + i += 1 else: break return i - @specialize.arg(1) - def _raise(self, msg, *args): - raise oefmt(self.space.w_ValueError, msg, *args) - def decode_any(self, i): i = self.skip_whitespace(i) ch = self.ll_chars[i] @@ -102,6 +149,11 @@ self._raise("No JSON object could be decoded: unexpected '%s' at char %d", ch, i) + + @specialize.arg(1) + def _raise(self, msg, *args): + raise oefmt(self.space.w_ValueError, msg, *args) + def decode_null(self, i): if (self.ll_chars[i] == 'u' and self.ll_chars[i+1] == 'l' and @@ -162,7 +214,7 @@ return self.decode_int_slow(start) self.pos = i - return self.space.newint(intval) + return self.intcache.newint(intval) def decode_float(self, i): from rpython.rlib import rdtoa @@ -214,6 +266,26 @@ ovf_maybe = (count >= OVF_DIGITS) return i, ovf_maybe, sign * intval + def _raise_control_char_in_string(self, ch, startindex, currindex): + if ch == '\0': + self._raise("Unterminated string starting at char %d", + startindex - 1) + else: + self._raise("Invalid control character at char %d", currindex-1) + + def _raise_object_error(self, ch, start, i): + if ch == '\0': + self._raise("Unterminated object starting at char %d", start) + else: + self._raise("Unexpected '%s' when decoding object (char %d)", + ch, i) + + def decode_surrogate_pair(self, i, highsurr): + """ uppon enter the following must hold: + chars[i] == "\\" and chars[i+1] == "u" + """ + # the possible ValueError is caught by the caller + def decode_array(self, i): w_list = self.space.newlist([]) start = i @@ -240,6 +312,13 @@ self._raise("Unexpected '%s' when decoding array (char %d)", ch, i-1) + def decode_any_context(self, i, context): + i = self.skip_whitespace(i) + ch = self.ll_chars[i] + if ch == '"': + return self.decode_string(i+1, context) + return self.decode_any(i) + def decode_object(self, i): start = i @@ -248,62 +327,124 @@ self.pos = i+1 return self.space.newdict() - d = self._create_empty_dict() + if self.scratch: + values_w = self.scratch.pop() + else: + values_w = [None] * self.DEFAULT_SIZE_SCRATCH + nextindex = 0 + currmap = self.startmap while True: # parse a key: value - w_name = self.decode_key(i) + currmap = self.decode_key(i, currmap) i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] if ch != ':': self._raise("No ':' found at char %d", i) i += 1 - i = self.skip_whitespace(i) - # - w_value = self.decode_any(i) - d[w_name] = w_value + + w_value = self.decode_any_context(i, currmap) + + if nextindex == len(values_w): # full + values_w = values_w + [None] * len(values_w) # double + values_w[nextindex] = w_value + nextindex += 1 i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] i += 1 if ch == '}': self.pos = i - return self._create_dict(d) + if currmap.is_blocked(): + currmap.instantiation_count += 1 + self.scratch.append(values_w) # can reuse next time + dict_w = self._switch_to_dict(currmap, values_w, nextindex) + return self._create_dict(dict_w) + self.scratch.append(values_w) # can reuse next time + values_w = values_w[:nextindex] + currmap.instantiation_count += 1 + w_res = self._create_dict_map(values_w, currmap) + if currmap.state != MapBase.USEFUL: + self.unclear_objects.append(w_res) + return w_res elif ch == ',': - pass - elif ch == '\0': - self._raise("Unterminated object starting at char %d", start) + i = self.skip_whitespace(i) + if currmap.is_blocked(): + currmap.instantiation_count += 1 + self.scratch.append(values_w) # can reuse next time + dict_w = self._switch_to_dict(currmap, values_w, nextindex) + return self.decode_object_dict(i, start, dict_w) else: - self._raise("Unexpected '%s' when decoding object (char %d)", - ch, i-1) + self._raise_object_error(ch, start, i - 1) - def decode_string(self, i): - start = i - bits = 0 + def _create_dict_map(self, values_w, jsonmap): + from pypy.objspace.std.jsondict import from_values_and_jsonmap + return from_values_and_jsonmap(self.space, values_w, jsonmap) + + def _devolve_jsonmap_dict(self, w_dict): + from pypy.objspace.std.jsondict import devolve_jsonmap_dict + devolve_jsonmap_dict(w_dict) + + def _get_jsonmap_from_dict(self, w_dict): + from pypy.objspace.std.jsondict import get_jsonmap_from_dict + return get_jsonmap_from_dict(w_dict) + + def _switch_to_dict(self, currmap, values_w, nextindex): + dict_w = self._create_empty_dict() + index = nextindex - 1 + while isinstance(currmap, JSONMap): + dict_w[currmap.w_key] = values_w[index] + index -= 1 + currmap = currmap.prev + assert len(dict_w) == nextindex + return dict_w + + def decode_object_dict(self, i, start, dict_w): while True: - # this loop is a fast path for strings which do not contain escape - # characters + # parse a key: value + w_key = self.decode_key_string(i) + i = self.skip_whitespace(self.pos) + ch = self.ll_chars[i] + if ch != ':': + self._raise("No ':' found at char %d", i) + i += 1 + + w_value = self.decode_any(i) + dict_w[w_key] = w_value + i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] i += 1 - bits |= ord(ch) - if ch == '"': + if ch == '}': self.pos = i - return self._create_string(start, i - 1, bits) - elif ch == '\\' or ch < '\x20': - self.pos = i-1 - return self.decode_string_escaped(start) + return self._create_dict(dict_w) + elif ch == ',': + i = self.skip_whitespace(i) + else: + self._raise_object_error(ch, start, i - 1) - def _create_string(self, start, end, bits): - if bits & 0x80: - # the 8th bit is set, it's an utf8 string - content_utf8 = self.getslice(start, end) + def decode_string_uncached(self, i): + start = i + ll_chars = self.ll_chars + nonascii, i = simd.find_end_of_string_no_hash(ll_chars, i, len(self.s)) + ch = ll_chars[i] + if ch == '\\': + self.pos = i + return self.decode_string_escaped(start, nonascii) + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + else: + assert ch == '"' + + self.pos = i + 1 + return self._create_string_wrapped(start, i, nonascii) + + def _create_string_wrapped(self, start, end, nonascii): + content = self.getslice(start, end) + if nonascii: + # contains non-ascii chars, we need to check that it's valid utf-8 lgt = unicodehelper.check_utf8_or_raise(self.space, - content_utf8) - return self.space.newutf8(content_utf8, lgt) + content) else: - # ascii only, fast path (ascii is a strict subset of - # latin1, and we already checked that all the chars are < - # 128) - return self.space.newutf8(self.getslice(start, end), - end - start) + lgt = end - start + return self.space.newutf8(content, lgt) def _create_dict(self, d): from pypy.objspace.std.dictmultiobject import from_unicode_key_dict @@ -313,8 +454,7 @@ from pypy.objspace.std.dictmultiobject import create_empty_unicode_key_dict return create_empty_unicode_key_dict(self.space) - - def decode_string_escaped(self, start): + def decode_string_escaped(self, start, nonascii): i = self.pos builder = StringBuilder((i - start) * 2) # just an estimate assert start >= 0 @@ -325,25 +465,21 @@ i += 1 if ch == '"': content_utf8 = builder.build() - lgt = unicodehelper.check_utf8_or_raise(self.space, + length = unicodehelper.check_utf8_or_raise(self.space, content_utf8) self.pos = i - return self.space.newutf8(content_utf8, lgt) + return self.space.newutf8(content_utf8, length) elif ch == '\\': - i = self.decode_escape_sequence(i, builder) + i = self.decode_escape_sequence_to_utf8(i, builder) elif ch < '\x20': - if ch == '\0': - self._raise("Unterminated string starting at char %d", - start - 1) - else: - self._raise("Invalid control character at char %d", i-1) + self._raise_control_char_in_string(ch, start, i) else: builder.append(ch) - def decode_escape_sequence(self, i, builder): + def decode_escape_sequence_to_utf8(self, i, stringbuilder): ch = self.ll_chars[i] i += 1 - put = builder.append + put = stringbuilder.append if ch == '\\': put('\\') elif ch == '"': put('"' ) elif ch == '/': put('/' ) @@ -353,22 +489,37 @@ elif ch == 'r': put('\r') elif ch == 't': put('\t') elif ch == 'u': - return self.decode_escape_sequence_unicode(i, builder) + # may be a suggorate pair + return self.decode_escape_sequence_unicode(i, stringbuilder) else: self._raise("Invalid \\escape: %s (char %d)", ch, i-1) return i + def _get_int_val_from_hex4(self, i): + ll_chars = self.ll_chars + res = 0 + for i in range(i, i + 4): + ch = ord(ll_chars[i]) + if ord('a') <= ch <= ord('f'): + digit = ch - ord('a') + 10 + elif ord('A') <= ch <= ord('F'): + digit = ch - ord('A') + 10 + elif ord('0') <= ch <= ord('9'): + digit = ch - ord('0') + else: + raise ValueError + res = (res << 4) + digit + return res + def decode_escape_sequence_unicode(self, i, builder): # at this point we are just after the 'u' of the \u1234 sequence. start = i i += 4 - hexdigits = self.getslice(start, i) try: - val = int(hexdigits, 16) + val = self._get_int_val_from_hex4(start) if (0xd800 <= val <= 0xdbff and self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u'): - hexdigits = self.getslice(i+2, i+6) - lowsurr = int(hexdigits, 16) + lowsurr = self._get_int_val_from_hex4(i + 2) if 0xdc00 <= lowsurr <= 0xdfff: # decode surrogate pair val = 0x10000 + (((val - 0xd800) << 10) | @@ -383,45 +534,534 @@ builder.append(utf8_ch) return i - def decode_key(self, i): - """ returns a wrapped unicode """ - from rpython.rlib.rarithmetic import intmask - i = self.skip_whitespace(i) + def decode_string(self, i, context=None): + ll_chars = self.ll_chars + start = i + ch = ll_chars[i] + if ch == '"': + self.pos = i + 1 + return self.w_empty_string # surprisingly common + + cache = True + if context is not None: + context.decoded_strings += 1 + if not context.should_cache(): + cache = False + if len(self.s) < self.MIN_SIZE_FOR_STRING_CACHE: + cache = False + + if not cache: + return self.decode_string_uncached(i) + + strhash, nonascii, i = simd.find_end_of_string(ll_chars, i, len(self.s)) + ch = ll_chars[i] + if ch == '\\': + self.pos = i + return self.decode_string_escaped(start, nonascii) + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + else: + assert ch == '"' + + self.pos = i + 1 + + length = i - start + strhash ^= length + + # check cache first: + try: + entry = self.cache_wrapped[strhash] + except KeyError: + w_res = self._create_string_wrapped(start, i, nonascii) + # only add *some* strings to the cache, because keeping them all is + # way too expensive + if ((context is not None and + context.decoded_strings < self.STRING_CACHE_EVALUATION_SIZE) or + strhash in self.lru_cache): + entry = WrappedCacheEntry( + self.getslice(start, start + length), w_res) + self.cache_wrapped[strhash] = entry + else: + self.lru_cache[self.lru_index] = strhash + self.lru_index = (self.lru_index + 1) & self.LRU_MASK + return w_res + if not _compare_cache_entry(self.space, entry.repr, ll_chars, start, length): + # hopefully rare + return self._create_string_wrapped(start, i, nonascii) + if context is not None: + context.cache_hits += 1 + return entry.w_uni + + def decode_key(self, i, currmap): + newmap = self._decode_key(i, currmap) + currmap.observe_transition(newmap) + return newmap + + def _decode_key(self, i, currmap): + ll_chars = self.ll_chars + nextmap = currmap.fast_path_key_parse(self, i) + if nextmap is not None: + return nextmap + + start = i + ch = ll_chars[i] + if ch != '"': + self._raise("Key name must be string at char %d", i) + i += 1 + w_key = self._decode_key_string(i) + return currmap.get_next(w_key, self.s, start, self.pos) + + def _decode_key_string(self, i): + ll_chars = self.ll_chars + start = i + + strhash, nonascii, i = simd.find_end_of_string(ll_chars, i, len(self.s)) + + ch = ll_chars[i] + if ch == '\\': + self.pos = i + w_key = self.decode_string_escaped(start, nonascii) + return w_key + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + length = i - start + strhash ^= length + self.pos = i + 1 + # check cache first: + try: + entry = self.cache[strhash] + except KeyError: + w_res = self._create_string_wrapped(start, i, nonascii) + entry = WrappedCacheEntry( + self.getslice(start, start + length), w_res) + self.cache[strhash] = entry + return w_res + if not _compare_cache_entry(self.space, entry.repr, ll_chars, start, length): + # hopefully rare + w_res = self._create_string_wrapped(start, i, nonascii) + print w_res + else: + w_res = entry.w_uni + return w_res + + def decode_key_string(self, i): ll_chars = self.ll_chars ch = ll_chars[i] if ch != '"': self._raise("Key name must be string at char %d", i) i += 1 + return self._decode_key_string(i) - start = i - bits = 0 - strhash = ord(ll_chars[i]) << 7 - while True: - ch = ll_chars[i] +class WrappedCacheEntry(object): + def __init__(self, repr, w_uni): + self.repr = repr + self.w_uni = w_uni + + +class MapBase(object): + # the basic problem we are trying to solve is the following: dicts in + # json can either be used as objects, or as dictionaries with arbitrary + # string keys. We want to use maps for the former, but not for the + # latter. But we don't know in advance which kind of dict is which. + + # Therefore we create "preliminary" maps where we aren't quite sure yet + # whether they are really useful maps or not. If we see them used often + # enough, we promote them to "useful" maps, which we will actually + # instantiate objects with. + + # If we determine that a map is not used often enough, we can turn it + # into a "blocked" map, which is a point in the map tree where we will + # switch to regular dicts, when we reach that part of the tree. + + # allowed graph edges or nodes in all_next: + # USEFUL ------- + # / \ \ + # v v v + # FRINGE USEFUL BLOCKED + # | + # v + # PRELIMINARY + # | + # v + # PRELIMINARY + + # state transitions: + # PRELIMINARY + # / | \ + # | v v + # | FRINGE -> USEFUL + # | | + # \ | + # v v + # BLOCKED + + # the single_nextmap edge can only be these graph edges: + # USEFUL + # | + # v + # USEFUL + # + # FRINGE + # | + # v + # PRELIMINARY + # | + # v + # PRELIMINARY + + USEFUL = 'u' + PRELIMINARY = 'p' + FRINGE = 'f' # buffer between PRELIMINARY and USEFUL + BLOCKED = 'b' + + # tunable parameters + MAX_FRINGE = 40 + USEFUL_THRESHOLD = 5 + + def __init__(self, space): + self.space = space + + # a single transition is stored in .single_nextmap + self.single_nextmap = None + + # all_next is only initialized after seeing the *second* transition + # but then it also contains .single_nextmap + self.all_next = None # later dict {key: nextmap} + + self.instantiation_count = 0 + self.number_of_leaves = 1 + + def get_terminator(self): + while isinstance(self, JSONMap): + self = self.prev + assert isinstance(self, Terminator) + return self + + def _check_invariants(self): + if self.all_next: + for next in self.all_next.itervalues(): + next._check_invariants() + elif self.single_nextmap: + self.single_nextmap._check_invariants() + + def get_next(self, w_key, string, start, stop): + from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq + if isinstance(self, JSONMap): + assert not self.state == MapBase.BLOCKED + single_nextmap = self.single_nextmap + if (single_nextmap is not None and + single_nextmap.w_key.eq_w(w_key)): + return single_nextmap + + assert stop >= 0 + assert start >= 0 + + if single_nextmap is None: + # first transition ever seen, don't initialize all_next + next = self._make_next_map(w_key, string[start:stop]) + self.single_nextmap = next + else: + if self.all_next is None: + self.all_next = objectmodel.r_dict(unicode_eq, unicode_hash, + force_non_null=True, simple_hash_eq=True) + self.all_next[single_nextmap.w_key] = single_nextmap + else: + next = self.all_next.get(w_key, None) + if next is not None: + return next + next = self._make_next_map(w_key, string[start:stop]) + self.all_next[w_key] = next + + # fix number_of_leaves + self.change_number_of_leaves(1) + + terminator = self.get_terminator() + terminator.register_potential_fringe(next) + return next + + def change_number_of_leaves(self, difference): + parent = self + while isinstance(parent, JSONMap): + parent.number_of_leaves += difference + parent = parent.prev + parent.number_of_leaves += difference # terminator + + def fast_path_key_parse(self, decoder, position): + single_nextmap = self.single_nextmap + if single_nextmap: + ll_chars = decoder.ll_chars + assert isinstance(single_nextmap, JSONMap) + if single_nextmap.key_repr_cmp(ll_chars, position): + decoder.pos = position + len(single_nextmap.key_repr) + return single_nextmap + + def observe_transition(self, newmap): + """ observe a transition from self to newmap. + This does a few things, including updating the self size estimate with + the knowledge that one object transitioned from self to newmap. + also it potentially decides that self should move to state USEFUL.""" + self.instantiation_count += 1 + if isinstance(self, JSONMap) and self.state == MapBase.FRINGE: + if self.is_useful(): + self.mark_useful() + + def _make_next_map(self, w_key, key_repr): + return JSONMap(self.space, self, w_key, key_repr) + + def _all_dot(self, output): + identity = objectmodel.compute_unique_id(self) + output.append('%s [shape=box%s];' % (identity, self._get_dot_text())) + if self.all_next: + for w_key, value in self.all_next.items(): + assert isinstance(value, JSONMap) + if value is self.single_nextmap: + color = ", color=blue" + else: + color = "" + output.append('%s -> %s [label="%s"%s];' % ( + identity, objectmodel.compute_unique_id(value), value.w_key._utf8, color)) + value._all_dot(output) + elif self.single_nextmap is not None: + value = self.single_nextmap + output.append('%s -> %s [label="%s", color=blue];' % ( + identity, objectmodel.compute_unique_id(value), value.w_key._utf8)) + value._all_dot(output) + + + def _get_dot_text(self): + return ", label=base" + + def view(self): + from dotviewer import graphclient + import pytest + r = ["digraph G {"] + self._all_dot(r) + r.append("}") + p = pytest.ensuretemp("resilientast").join("temp.dot") + p.write("\n".join(r)) + graphclient.display_dot_file(str(p)) + + def _get_caching_stats(self): + caching = 0 + num_maps = 1 + if isinstance(self, JSONMap) and self.should_cache() and self.decoded_strings > 200: + caching += 1 + + if self.all_next: + children = self.all_next.values() + elif self.single_nextmap: + children = [self.single_nextmap] + else: + children = [] + for child in children: + a, b = child._get_caching_stats() + caching += a + num_maps += b + return caching, num_maps + +class Terminator(MapBase): + def __init__(self, space): + MapBase.__init__(self, space) + self.all_object_count = 0 + self.current_fringe = {} + + def register_potential_fringe(self, prelim): + prev = prelim.prev + if (isinstance(prev, Terminator) or + isinstance(prev, JSONMap) and prev.state == MapBase.USEFUL): + prelim.state = MapBase.FRINGE + + if len(self.current_fringe) > MapBase.MAX_FRINGE: + self.cleanup_fringe() + self.current_fringe[prelim] = None + + def cleanup_fringe(self): + min_fringe = None + min_avg = 10000000000 + for f in self.current_fringe: + if f.state == MapBase.FRINGE: + avg = f.average_instantiation() + if avg < min_avg: + min_avg = avg + min_fringe = f + else: + for f in self.current_fringe.keys(): + if f.state != MapBase.FRINGE: + del self.current_fringe[f] + return + assert min_fringe + min_fringe.mark_blocked() + del self.current_fringe[min_fringe] + + +class JSONMap(MapBase): + """ A map implementation to speed up parsing """ + + def __init__(self, space, prev, w_key, key_repr): + MapBase.__init__(self, space) + + self.prev = prev + self.w_key = w_key + self.key_repr = key_repr + + self.state = MapBase.PRELIMINARY + + # key decoding stats + self.decoded_strings = 0 + self.cache_hits = 0 + + # for jsondict support + self.key_to_index = None + self.keys_in_order = None + self.strategy_instance = None + + @jit.elidable + def get_terminator(self): + while isinstance(self, JSONMap): + self = self.prev + assert isinstance(self, Terminator) + return self + + def _check_invariants(self): + assert self.state in ( + MapBase.USEFUL, + MapBase.PRELIMINARY, + MapBase.FRINGE, + MapBase.BLOCKED, + ) + + prev = self.prev + if isinstance(prev, JSONMap): + prevstate = prev.state + else: + prevstate = MapBase.USEFUL + + if prevstate == MapBase.USEFUL: + assert self.state != MapBase.PRELIMINARY + elif prevstate == MapBase.PRELIMINARY: + assert self.state == MapBase.PRELIMINARY + elif prevstate == MapBase.FRINGE: + assert self.state == MapBase.PRELIMINARY + else: + # if prevstate is BLOCKED, we shouldn't have recursed here! + assert False, "should be unreachable" + + if self.state == MapBase.BLOCKED: + assert self.single_nextmap is None + assert self.all_next is None + + MapBase._check_invariants(self) + + def mark_useful(self): + # mark self as useful, and also the most commonly instantiated + # children, recursively + assert self.state in (MapBase.FRINGE, MapBase.PRELIMINARY) + self.state = MapBase.USEFUL + maxchild = self.single_nextmap + if self.all_next is not None: + for child in self.all_next.itervalues(): + if child.instantiation_count > maxchild.instantiation_count: + maxchild = child + if maxchild is not None: + maxchild.mark_useful() + if self.all_next: + terminator = self.get_terminator() + for child in self.all_next.itervalues(): + if child is not maxchild: + terminator.register_potential_fringe(child) + self.single_nextmap = maxchild + + def mark_blocked(self): + self.state = MapBase.BLOCKED + if self.all_next: + for next in self.all_next.itervalues(): + next.mark_blocked() + elif self.single_nextmap: + self.single_nextmap.mark_blocked() + self.single_nextmap = None + self.all_next = None + self.change_number_of_leaves(-self.number_of_leaves + 1) + + def is_blocked(self): + return self.state == MapBase.BLOCKED + + def average_instantiation(self): + return self.instantiation_count / float(self.number_of_leaves) + + def is_useful(self): + return self.average_instantiation() > self.USEFUL_THRESHOLD + + def should_cache(self): + return not (self.decoded_strings > JSONDecoder.STRING_CACHE_EVALUATION_SIZE and + self.cache_hits * JSONDecoder.STRING_CACHE_USEFULNESS_FACTOR < self.decoded_strings) + + def key_repr_cmp(self, ll_chars, i): + for j, c in enumerate(self.key_repr): + if ll_chars[i] != c: + return False i += 1 - if ch == '"': - break - elif ch == '\\' or ch < '\x20': - self.pos = i-1 - return self.decode_string_escaped(start) - strhash = intmask((1000003 * strhash) ^ ord(ll_chars[i])) - bits |= ord(ch) - length = i - start - 1 - if length == 0: - strhash = -1 + return True + + # _____________________________________________________ + # methods for JsonDictStrategy + + @jit.elidable + def get_index(self, w_key): + from pypy.objspace.std.unicodeobject import W_UnicodeObject + assert isinstance(w_key, W_UnicodeObject) + return self.get_key_to_index().get(w_key, -1) + + def get_key_to_index(self): + from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq + key_to_index = self.key_to_index + if key_to_index is None: + key_to_index = self.key_to_index = objectmodel.r_dict(unicode_eq, unicode_hash, + force_non_null=True, simple_hash_eq=True) + # compute depth + curr = self + depth = 0 + while True: + depth += 1 + curr = curr.prev + if not isinstance(curr, JSONMap): + break + + curr = self + while depth: + depth -= 1 + key_to_index[curr.w_key] = depth + curr = curr.prev + if not isinstance(curr, JSONMap): + break + return key_to_index + + def get_keys_in_order(self): + keys_in_order = self.keys_in_order + if keys_in_order is None: + key_to_index = self.get_key_to_index() + keys_in_order = self.keys_in_order = [None] * len(key_to_index) + for w_key, index in key_to_index.iteritems(): + keys_in_order[index] = w_key + return keys_in_order + + # _____________________________________________________ + + def _get_dot_text(self): + if self.all_next is None: + l = int(self.single_nextmap is not None) else: - strhash ^= length - strhash = intmask(strhash) - self.pos = i - # check cache first: - key = (ll_chars, start, length, strhash) - try: - return self.cache[key] - except KeyError: - pass - res = self._create_string(start, i - 1, bits) - self.cache[key] = res + l = len(self.all_next) + 1 + extra = "" + if self.decoded_strings: + extra = "\\n%s/%s (%s%%)" % (self.cache_hits, self.decoded_strings, self.cache_hits/float(self.decoded_strings)) + res = ', label="#%s\\nchildren: %s%s"' % (self.instantiation_count, l, extra) + if self.state == MapBase.BLOCKED: + res += ", fillcolor=lightsalmon" + if self.state == MapBase.FRINGE: + res += ", fillcolor=lightgray" + if self.state == MapBase.PRELIMINARY: + res += ", fillcolor=lightslategray" return res @@ -442,3 +1082,4 @@ return w_res finally: decoder.close() + diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py new file mode 100644 --- /dev/null +++ b/pypy/module/_pypyjson/simd.py @@ -0,0 +1,218 @@ +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import objectmodel, unroll +from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT +from rpython.jit.backend.detect_cpu import autodetect + +# accelerators for string operations using simd on regular word sizes (*not* +# SSE instructions). this style is sometimes called SWAR (SIMD Within A +# Register) or "broadword techniques" + +# XXX remove wordsize and endianness restrictions properly, so far only x86-64 +# is tested + +USE_SIMD = False +if LONG_BIT == 64: + WORD_SIZE = 8 + EVERY_BYTE_ONE = 0x0101010101010101 + EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080 + if autodetect() == "x86-64": + USE_SIMD = True +else: + WORD_SIZE = 4 + EVERY_BYTE_ONE = 0x01010101 + EVERY_BYTE_HIGHEST_BIT = 0x80808080 + + +# helpers + +unrolling_wordsize = unroll.unrolling_iterable(range(WORD_SIZE)) + +def char_repeated_word_width(ch): + return r_uint(EVERY_BYTE_ONE) * ord(ch) + +def any_char_zero(word): + return (word - r_uint(EVERY_BYTE_ONE)) & ~word & r_uint(EVERY_BYTE_HIGHEST_BIT) + +def any_char_in_words_zero(*words): + return _any_char_in_any_word_zero_accum(0, *words) + +def _any_char_in_any_word_zero_accum(accum, word, *words): + accum |= (word - r_uint(EVERY_BYTE_ONE)) & ~word + if not words: + return accum & r_uint(EVERY_BYTE_HIGHEST_BIT) + return _any_char_in_any_word_zero_accum(accum, *words) + +def print_chars(word): + # for debugging + out = '' + for i in range(WORD_SIZE): + out += chr(word & 0xff) + word >>= 8 + return out + +def index_nonzero(word): + # XXX can be done very cheap in theory + assert word + for i in unrolling_wordsize: + if word & 0xff: + return i + word >>= 8 + assert 0 + +def index_zero(word): + # XXX can be done very cheap in theory + assert any_char_zero(word) + for i in unrolling_wordsize: + if not word & 0xff: + return i + word >>= 8 + assert 0 # XXX ??? + +def splice_words(word, offset, other): + mask = ((~r_uint(0)) << (8 * offset)) + return (word & mask) | (other & ~mask) + + + + at objectmodel.always_inline +def position_string_ender(word): + maskquote = char_repeated_word_width('"') + maskbackslash = char_repeated_word_width('\\') + maskx20 = char_repeated_word_width(chr(0xff - 0x1f)) + # x1 and x2 check for equality, if a byte is 0 the corresponding + # char is equal to " or \ + x1 = maskquote ^ word + x2 = maskbackslash ^ word + # x3 checks for char < 0x20, the byte is 0 in that case + x3 = maskx20 & word + return any_char_in_words_zero(x1, x2, x3) + + at objectmodel.always_inline +def find_end_of_string_simd_unaligned(ll_chars, startpos, length): + ch = ll_chars[startpos] + strhash = (ord(ch) << 7) ^ 0x345678 + + wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) + num_safe_reads = (length - startpos) // WORD_SIZE + + bits = 0 + for i in range(num_safe_reads): + word = wordarray[i] + cond = position_string_ender(word) + if cond: + break + bits |= word + strhash = intmask((1000003 * strhash) ^ intmask(word)) + else: + # didn't find end of string yet, look at remaining chars + word = 0 + shift = 0 + i = 0 + for i in range(num_safe_reads * WORD_SIZE + startpos, length + 1): + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + bits |= ord(ch) + word |= ord(ch) << shift + shift += 8 + if shift: + strhash = intmask((1000003 * strhash) ^ intmask(word)) + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + return strhash, nonascii, i + + # compute endposition + nonzero = index_nonzero(cond) + endposition = startpos + i * WORD_SIZE + nonzero + if nonzero: + word = splice_words(r_uint(0), nonzero, word) + bits |= word + strhash = intmask((1000003 * strhash) ^ intmask(word)) + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + + return strhash, nonascii, endposition + + at objectmodel.always_inline +def find_end_of_string_simd_unaligned_no_hash(ll_chars, startpos, length): + ch = ll_chars[startpos] + + wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) + num_safe_reads = (length - startpos) // WORD_SIZE + + bits = 0 + for i in range(num_safe_reads): + word = wordarray[i] + cond = position_string_ender(word) + if cond: + break + bits |= word + else: + # didn't find end of string yet, look at remaining chars + word = 0 + shift = 0 + i = 0 + for i in range(num_safe_reads * WORD_SIZE + startpos, length + 1): + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + bits |= ord(ch) + word |= ord(ch) << shift + shift += WORD_SIZE + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + return nonascii, i + + # compute endposition + nonzero = index_nonzero(cond) + endposition = startpos + i * WORD_SIZE + nonzero + if nonzero: + word = splice_words(r_uint(0), nonzero, word) + bits |= word + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + + return nonascii, endposition + + + at objectmodel.always_inline +def find_end_of_string_slow(ll_chars, i, length): + ch = ll_chars[i] + strhash = (ord(ch) << 7) ^ 0x345678 + word = 0 + shift = 0 + + bits = 0 + + while True: + # this loop is a fast path for strings which do not contain escape + # characters + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + i += 1 + bits |= ord(ch) + + word |= ord(ch) << shift + shift += WORD_SIZE + if shift == WORD_SIZE * WORD_SIZE: + strhash = intmask((1000003 * strhash) ^ word) + shift = 0 + word = 0 + + if shift: + strhash = intmask((1000003 * strhash) ^ word) + return strhash, bool(bits & 0x80), i + +if USE_SIMD: + find_end_of_string = find_end_of_string_simd_unaligned + find_end_of_string_no_hash = find_end_of_string_simd_unaligned_no_hash +else: + find_end_of_string = find_end_of_string_slow + + @objectmodel.always_inline + def find_end_of_string_no_hash(ll_chars, i, length): + _, nonascii, i = find_end_of_string_slow(ll_chars, i, length) + return (nonascii, i) + + diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -1,31 +1,253 @@ # -*- encoding: utf-8 -*- -from pypy.module._pypyjson.interp_decoder import JSONDecoder +import pytest +from pypy.module._pypyjson.interp_decoder import JSONDecoder, Terminator, MapBase +from rpython.rtyper.lltypesystem import lltype, rffi -def test_skip_whitespace(): - s = ' hello ' - dec = JSONDecoder('fake space', s) - assert dec.pos == 0 - assert dec.skip_whitespace(0) == 3 - assert dec.skip_whitespace(3) == 3 - assert dec.skip_whitespace(8) == len(s) - dec.close() -class FakeSpace(object): - def newutf8(self, s, l): - return s +class TestJson(object): + def test_skip_whitespace(self): + s = ' hello ' + dec = JSONDecoder(self.space, s) + assert dec.pos == 0 + assert dec.skip_whitespace(0) == 3 + assert dec.skip_whitespace(3) == 3 + assert dec.skip_whitespace(8) == len(s) + dec.close() -def test_decode_key(): - s1 = "123" * 100 - s = ' "%s" "%s" ' % (s1, s1) - dec = JSONDecoder(FakeSpace(), s) - assert dec.pos == 0 - x = dec.decode_key(0) - assert x == s1 - # check caching - y = dec.decode_key(dec.pos) - assert y == s1 - assert y is x - dec.close() + def test_json_map(self): + m = Terminator(self.space) + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + m1 = m.get_next(w_a, '"a"', 0, 3) + assert m1.w_key == w_a + assert m1.single_nextmap is None + assert m1.key_repr == '"a"' + assert m1.key_repr_cmp('"a": 123', 0) + assert not m1.key_repr_cmp('b": 123', 0) + assert m.single_nextmap.w_key == w_a + + m2 = m.get_next(w_a, '"a"', 0, 3) + assert m2 is m1 + + m3 = m.get_next(w_b, '"b"', 0, 3) + assert m3.w_key == w_b + assert m3.single_nextmap is None + assert m3.key_repr == '"b"' + assert m.single_nextmap is m1 + + m4 = m3.get_next(w_c, '"c"', 0, 3) + assert m4.w_key == w_c + assert m4.single_nextmap is None + assert m4.key_repr == '"c"' + assert m3.single_nextmap is m4 + + def test_json_map_get_index(self): + m = Terminator(self.space) + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + m1 = m.get_next(w_a, 'a"', 0, 2) + assert m1.get_index(w_a) == 0 + assert m1.get_index(w_b) == -1 + + m2 = m.get_next(w_b, 'b"', 0, 2) + assert m2.get_index(w_b) == 0 + assert m2.get_index(w_a) == -1 + + m3 = m2.get_next(w_c, 'c"', 0, 2) + assert m3.get_index(w_b) == 0 + assert m3.get_index(w_c) == 1 + assert m3.get_index(w_a) == -1 + + def test_decode_key(self): + m = Terminator(self.space) + m_diff = Terminator(self.space) + for s1 in ["abc", "1001" * 10, u"ä".encode("utf-8")]: + s = ' "%s" "%s" "%s"' % (s1, s1, s1) + dec = JSONDecoder(self.space, s) + assert dec.pos == 0 + m1 = dec.decode_key(dec.skip_whitespace(0), m) + assert m1.w_key._utf8 == s1 + assert m1.key_repr == '"%s"' % s1 + + # check caching on w_key level + m2 = dec.decode_key(dec.skip_whitespace(dec.pos), m_diff) + assert m1.w_key is m2.w_key + + # check caching on map level + m3 = dec.decode_key(dec.skip_whitespace(dec.pos), m_diff) + assert m3 is m2 + dec.close() + + def test_decode_string_caching(self): + for s1 in ["abc", u"ä".encode("utf-8")]: + s = '"%s" "%s" "%s"' % (s1, s1, s1) + dec = JSONDecoder(self.space, s) + dec.MIN_SIZE_FOR_STRING_CACHE = 0 + assert dec.pos == 0 + w_x = dec.decode_string(1) + w_y = dec.decode_string(dec.skip_whitespace(dec.pos) + 1) + assert w_x is not w_y + # check caching + w_z = dec.decode_string(dec.skip_whitespace(dec.pos) + 1) + assert w_z is w_y + dec.close() + + def _make_some_maps(self): + # base -> m1 -> m2 -> m3 + # \-> m4 + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + w_d = self.space.newutf8("d", 1) + base = Terminator(self.space) + base.instantiation_count = 6 + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = m1.get_next(w_b, 'b"', 0, 2) + m3 = m2.get_next(w_c, 'c"', 0, 2) + m4 = m2.get_next(w_d, 'd"', 0, 2) + return base, m1, m2, m3, m4 + + # unit tests for map state transistions + def test_fringe_to_useful(self): + base, m1, m2, m3, m4 = self._make_some_maps() + base.instantiation_count = 6 + assert m1.state == MapBase.FRINGE + m1.instantiation_count = 6 + + assert m2.state == MapBase.PRELIMINARY + m2.instantiation_count = 6 + + assert m3.state == MapBase.PRELIMINARY + m3.instantiation_count = 2 + assert m2.single_nextmap is m3 + + assert m4.state == MapBase.PRELIMINARY + m4.instantiation_count = 4 + + m1.mark_useful() + assert m1.state == MapBase.USEFUL + assert m2.state == MapBase.USEFUL + assert m3.state == MapBase.FRINGE + assert m4.state == MapBase.USEFUL + assert m2.single_nextmap is m4 + + assert m1.number_of_leaves == 2 + base._check_invariants() + + def test_number_of_leaves(self): + w_x = self.space.newutf8("x", 1) + base, m1, m2, m3, m4 = self._make_some_maps() + assert base.number_of_leaves == 2 + assert m1.number_of_leaves == 2 + assert m2.number_of_leaves == 2 + assert m3.number_of_leaves == 1 + assert m4.number_of_leaves == 1 + m5 = m2.get_next(w_x, 'x"', 0, 2) + assert base.number_of_leaves == 3 + assert m1.number_of_leaves == 3 + assert m2.number_of_leaves == 3 + assert m5.number_of_leaves == 1 + + def test_cleanup_fringe_simple(self): + base, m1, m2, m3, m4 = self._make_some_maps() + base.instantiation_count = 6 + assert m1.state == MapBase.FRINGE + m1.instantiation_count = 6 + m2.instantiation_count = 6 + m3.instantiation_count = 2 + m4.instantiation_count = 4 + assert base.current_fringe == {m1: None} + + m1.mark_useful() + assert base.current_fringe == {m1: None, m3: None} # not cleaned up + base.cleanup_fringe() + assert base.current_fringe == {m3: None} + + def test_cleanup_fringe_block(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + w_d = self.space.newutf8("d", 1) + base = Terminator(self.space) + base.instantiation_count = 6 + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = base.get_next(w_b, 'b"', 0, 2) + m3 = base.get_next(w_c, 'c"', 0, 2) + m4 = base.get_next(w_d, 'd"', 0, 2) + m5 = m4.get_next(w_a, 'a"', 0, 2) + base.instantiation_count = 7 + m1.instantiation_count = 2 + m2.instantiation_count = 2 + m3.instantiation_count = 2 + m4.instantiation_count = 1 + m5.instantiation_count = 1 + assert base.current_fringe == dict.fromkeys([m1, m2, m3, m4]) + + base.cleanup_fringe() + assert base.current_fringe == dict.fromkeys([m1, m2, m3]) + assert m4.state == MapBase.BLOCKED + assert m4.single_nextmap is None + assert m4.all_next is None + assert m5.state == MapBase.BLOCKED + assert m5.single_nextmap is None + assert m5.all_next is None + + def test_deal_with_blocked(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + space = self.space + s = '{"a": 1, "b": 2, "c": 3}' + dec = JSONDecoder(space, s) + dec.startmap = base = Terminator(space) + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = m1.get_next(w_b, 'b"', 0, 2) + m2.mark_blocked() + w_res = dec.decode_object(1) + assert space.int_w(space.len(w_res)) == 3 + assert space.int_w(space.getitem(w_res, w_a)) == 1 + assert space.int_w(space.getitem(w_res, w_b)) == 2 + assert space.int_w(space.getitem(w_res, w_c)) == 3 + dec.close() + + def test_deal_with_blocked_number_of_leaves(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_x = self.space.newutf8("x", 1) + w_u = self.space.newutf8("u", 1) + space = self.space + base = Terminator(space) + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = m1.get_next(w_b, 'b"', 0, 2) + m2.get_next(w_x, 'x"', 0, 2) + m2.get_next(w_u, 'u"', 0, 2) + assert base.number_of_leaves == 2 + m2.mark_blocked() + assert base.number_of_leaves == 1 + + @pytest.mark.skip() + def test_caching_stats(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_x = self.space.newutf8("x", 1) + w_u = self.space.newutf8("u", 1) + space = self.space + base = Terminator(space) + m1 = base.get_next(w_a, 'a"', 0, 2) + m2 = m1.get_next(w_b, 'b"', 0, 2) + m2.get_next(w_x, 'x"', 0, 2) + m2.get_next(w_u, 'u"', 0, 2) + m1.decode_string = 300 + m1.cache_hits = 0 + m3 = base.get_next(w_b, '"b"', 0, 3) + m3.decode_string = 300 + m3.cache_hits = 300 + caching_maps, total_maps = base._get_caching_stats() + assert caching_maps == 5 + assert total_maps == 6 + class AppTest(object): spaceconfig = {"objspace.usemodules._pypyjson": True} @@ -55,7 +277,7 @@ raises(ValueError, _pypyjson.loads, 'fa') raises(ValueError, _pypyjson.loads, 'f') raises(ValueError, _pypyjson.loads, 'falXX') - + def test_decode_string(self): import _pypyjson @@ -85,7 +307,7 @@ import _pypyjson assert _pypyjson.loads(r'"\\"') == u'\\' assert _pypyjson.loads(r'"\""') == u'"' - assert _pypyjson.loads(r'"\/"') == u'/' + assert _pypyjson.loads(r'"\/"') == u'/' assert _pypyjson.loads(r'"\b"') == u'\b' assert _pypyjson.loads(r'"\f"') == u'\f' assert _pypyjson.loads(r'"\n"') == u'\n' @@ -101,12 +323,19 @@ import _pypyjson s = r'"hello\nworld' # missing the trailing " raises(ValueError, "_pypyjson.loads(s)") - + def test_escape_sequence_unicode(self): import _pypyjson s = r'"\u1234"' assert _pypyjson.loads(s) == u'\u1234' + def test_escape_sequence_mixed_with_utf8(self): + import _pypyjson + utf8 = u'ä"'.encode("utf-8") + assert _pypyjson.loads(r'"abc\\' + utf8) == u'abc\\ä' + assert _pypyjson.loads(r'"abc\"' + utf8) == u'abc"ä' + assert _pypyjson.loads(r'"def\u1234' + utf8) == u'def\u1234ä' + def test_invalid_utf_8(self): import _pypyjson s = '"\xe0"' # this is an invalid UTF8 sequence inside a string @@ -176,13 +405,18 @@ s = '{"hello": "world", "aaa": "bbb"}' assert _pypyjson.loads(s) == {'hello': 'world', 'aaa': 'bbb'} + assert _pypyjson.loads(s) == {'hello': 'world', + 'aaa': 'bbb'} raises(ValueError, _pypyjson.loads, '{"key"') raises(ValueError, _pypyjson.loads, '{"key": 42') + assert _pypyjson.loads('{"neighborhood": ""}') == { + "neighborhood": ""} + def test_decode_object_nonstring_key(self): import _pypyjson raises(ValueError, "_pypyjson.loads('{42: 43}')") - + def test_decode_array(self): import _pypyjson assert _pypyjson.loads('[]') == [] @@ -263,3 +497,4 @@ for inputtext, errmsg in test_cases: exc = raises(ValueError, _pypyjson.loads, inputtext) assert str(exc.value) == errmsg + diff --git a/pypy/module/_pypyjson/test/test_simd.py b/pypy/module/_pypyjson/test/test_simd.py new file mode 100644 --- /dev/null +++ b/pypy/module/_pypyjson/test/test_simd.py @@ -0,0 +1,104 @@ +import sys +import pytest +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import r_uint, intmask + +from pypy.module._pypyjson.simd import USE_SIMD +from pypy.module._pypyjson.simd import find_end_of_string_slow +from pypy.module._pypyjson.simd import print_chars +from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned, WORD_SIZE +from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned_no_hash + +from hypothesis import example, given, strategies + +if not USE_SIMD: + pytest.skip("only implemented for 64 bit for now") + +def fill_to_word_size(res, ch=" "): + if len(res) % WORD_SIZE != 0: + res += ch * (WORD_SIZE - (len(res) % WORD_SIZE)) + return res + +def string_to_word(s): + assert len(s) == WORD_SIZE + ll_chars, flag = rffi.get_nonmovingbuffer_final_null(s) + try: + wordarray = rffi.cast(rffi.ULONGP, ll_chars) + return wordarray[0] + finally: + rffi.free_nonmovingbuffer(s, ll_chars, flag) + +def ll(callable, string, *args): + ll_chars, flag = rffi.get_nonmovingbuffer_final_null(string) + try: + return callable(ll_chars, *args) + finally: + rffi.free_nonmovingbuffer(string, ll_chars, flag) + +word = strategies.builds( + r_uint, strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint)) + +def build_string(prefix, content, end, suffix): + res = prefix + '"' + "".join([chr(x) for x in content]) + end + suffix + return fill_to_word_size(res), len(prefix) + 1 + +string_in_context_strategy = strategies.builds( + build_string, prefix=strategies.binary(), + content=strategies.lists(strategies.integers(1, 255), min_size=1), + end=strategies.sampled_from('"\\\x00\x01'), + suffix=strategies.binary()) + +def compare(string, res1, res2): + hash1, nonascii1, endindex1 = res1 + hash2, nonascii2, endindex2 = res2 + assert endindex1 == endindex2 + if string[endindex1 - 1] == '"': + assert hash1 == hash2 + assert nonascii1 == nonascii2 + + + at example(('" \x80" ', 1)) + at example(('"\x01" ', 1)) + at example(('"aaaaaaaa"\x00\x00\x00\x00\x00\x00\x00 ', 1)) + at example(('"aaaaaaaa" ', 1)) + at example(('"12"', 1)) + at example(('"1234567abcdefghAB"', 1)) + at example(('"1234567abcdefgh"', 1)) + at example((' "123456ABCDEF" \x00', 2)) + at example((' "123456aaaaaaaaABCDEF"\x00', 2)) + at given(string_in_context_strategy) +def test_find_end_of_string(a): + (string, startindex) = a + res = ll(find_end_of_string_slow, string, startindex, len(string)) + hash, nonascii1, endposition1 = res + ch = string[endposition1] + assert ch == '"' or ch == '\\' or ch < '\x20' + for ch in string[startindex:endposition1]: + assert not (ch == '"' or ch == '\\' or ch < '\x20') + compare(string, res, ll(find_end_of_string_simd_unaligned, string, startindex, len(string))) + + nonascii2, endposition2 = ll(find_end_of_string_simd_unaligned_no_hash, string, startindex, len(string)) + assert nonascii1 == nonascii2 + assert endposition1 == endposition2 + + at given(string_in_context_strategy, strategies.binary(min_size=1)) +def test_find_end_of_string_position_invariance(a, prefix): + fn = find_end_of_string_simd_unaligned + (string, startindex) = a + h1, nonascii1, i1 = ll(fn, string, startindex, len(string)) + string2 = prefix + string + h2, nonascii2, i2 = ll(fn, string2, startindex + len(prefix), len(string) + len(prefix)) + assert h1 == h2 + assert nonascii1 == nonascii2 + assert i1 + len(prefix) == i2 + + at given(string_in_context_strategy, strategies.binary(min_size=1)) +def test_find_end_of_string_position_invariance_no_hash(a, prefix): + fn = find_end_of_string_simd_unaligned_no_hash + (string, startindex) = a + nonascii1, i1 = ll(fn, string, startindex, len(string)) + string2 = prefix + string + nonascii2, i2 = ll(fn, string2, startindex + len(prefix), len(string) + len(prefix)) + assert nonascii1 == nonascii2 + assert i1 + len(prefix) == i2 + diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py new file mode 100644 --- /dev/null +++ b/pypy/objspace/std/jsondict.py @@ -0,0 +1,167 @@ +"""dict implementation specialized for object loaded by the _pypyjson module. + +Somewhat similar to MapDictStrategy, also uses a map. +""" + +from rpython.rlib import jit, rerased, objectmodel, debug + +from pypy.objspace.std.dictmultiobject import ( + UnicodeDictStrategy, DictStrategy, + create_iterator_classes, W_DictObject) + + +def from_values_and_jsonmap(space, values_w, jsonmap): + if not objectmodel.we_are_translated(): + assert len(values_w) == len(jsonmap.get_keys_in_order()) + assert len(values_w) != 0 + debug.make_sure_not_resized(values_w) + strategy = jsonmap.strategy_instance + if strategy is None: + jsonmap.strategy_instance = strategy = JsonDictStrategy(space, jsonmap) + storage = strategy.erase(values_w) + return W_DictObject(space, strategy, storage) + +def devolve_jsonmap_dict(w_dict): + assert isinstance(w_dict, W_DictObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, JsonDictStrategy) + strategy.switch_to_unicode_strategy(w_dict) + +def get_jsonmap_from_dict(w_dict): + assert isinstance(w_dict, W_DictObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, JsonDictStrategy) + return strategy.jsonmap + +class JsonDictStrategy(DictStrategy): + erase, unerase = rerased.new_erasing_pair("jsondict") + erase = staticmethod(erase) + unerase = staticmethod(unerase) + + _immutable_fields_ = ['jsonmap'] + + def __init__(self, space, jsonmap): + DictStrategy.__init__(self, space) + self.jsonmap = jsonmap + + def wrap(self, w_key): + return w_key + + def wrapkey(space, key): + return key + + def get_empty_storage(self): + raise NotImplementedError("should not be reachable") + + def is_correct_type(self, w_obj): + space = self.space + return space.is_w(space.type(w_obj), space.w_unicode) + + def _never_equal_to(self, w_lookup_type): + return False + + def length(self, w_dict): + return len(self.unerase(w_dict.dstorage)) + + def getitem(self, w_dict, w_key): + if self.is_correct_type(w_key): + return self.getitem_unicode(w_dict, w_key) + else: + self.switch_to_unicode_strategy(w_dict) + return w_dict.getitem(w_key) + + def getitem_unicode(self, w_dict, w_key): + storage_w = self.unerase(w_dict.dstorage) + if jit.isconstant(w_key): + jit.promote(self) + index = self.jsonmap.get_index(w_key) + if index == -1: + return None + return storage_w[index] + + def setitem(self, w_dict, w_key, w_value): + if self.is_correct_type(w_key): + storage_w = self.unerase(w_dict.dstorage) + index = self.jsonmap.get_index(w_key) + if index != -1: + storage_w[index] = w_value + return + self.switch_to_unicode_strategy(w_dict) + w_dict.setitem(w_key, w_value) + + def setdefault(self, w_dict, w_key, w_default): + if self.is_correct_type(w_key): + w_result = self.getitem_unicode(w_dict, w_key) + if w_result is not None: + return w_result + self.switch_to_unicode_strategy(w_dict) + return w_dict.setdefault(w_key, w_default) + + def delitem(self, w_dict, w_key): + self.switch_to_unicode_strategy(w_dict) + return w_dict.delitem(w_key) + + def popitem(self, w_dict): + self.switch_to_unicode_strategy(w_dict) + return w_dict.popitem() + + def switch_to_unicode_strategy(self, w_dict): + strategy = self.space.fromcache(UnicodeDictStrategy) + values_w = self.unerase(w_dict.dstorage) + storage = strategy.get_empty_storage() + d_new = strategy.unerase(storage) + keys_in_order = self.jsonmap.get_keys_in_order() + assert len(keys_in_order) == len(values_w) + for index, w_key in enumerate(keys_in_order): + assert w_key is not None + assert type(w_key) is self.space.UnicodeObjectCls + d_new[w_key] = values_w[index] + w_dict.set_strategy(strategy) + w_dict.dstorage = storage + + def w_keys(self, w_dict): + return self.space.newlist(self.jsonmap.get_keys_in_order()) + + def values(self, w_dict): + return self.unerase(w_dict.dstorage)[:] # to make resizable + + def items(self, w_dict): + space = self.space + storage_w = self.unerase(w_dict.dstorage) + res = [None] * len(storage_w) + for index, w_key in enumerate(self.jsonmap.get_keys_in_order()): + res[index] = space.newtuple([w_key, storage_w[index]]) + return res + + def getiterkeys(self, w_dict): + return iter(self.jsonmap.get_keys_in_order()) + + def getitervalues(self, w_dict): + storage_w = self.unerase(w_dict.dstorage) + return iter(storage_w) + + def getiteritems_with_hash(self, w_dict): + storage_w = self.unerase(w_dict.dstorage) + return ZipItemsWithHash(self.jsonmap.get_keys_in_order(), storage_w) + + +class ZipItemsWithHash(object): + def __init__(self, list1, list2): + assert len(list1) == len(list2) + self.list1 = list1 + self.list2 = list2 + self.i = 0 + + def __iter__(self): + return self + + def next(self): + i = self.i + if i >= len(self.list1): + raise StopIteration + self.i = i + 1 + w_key = self.list1[i] + return (w_key, self.list2[i], w_key.hash_w()) + + +create_iterator_classes(JsonDictStrategy) diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py new file mode 100644 --- /dev/null +++ b/pypy/objspace/std/test/test_jsondict.py @@ -0,0 +1,89 @@ + +class AppTest(object): + spaceconfig = {"objspace.usemodules._pypyjson": True} + + def test_check_strategy(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1}') + assert __pypy__.strategy(d) == "JsonDictStrategy" + d = _pypyjson.loads('{}') + assert __pypy__.strategy(d) == "EmptyDictStrategy" + + def test_simple(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert len(d) == 2 + assert d[u"a"] == 1 + assert d[u"b"] == u"x" + assert u"c" not in d + + d[u"a"] = 5 + assert d[u"a"] == 5 + assert __pypy__.strategy(d) == "JsonDictStrategy" + + # devolve it + assert not 1 in d + assert __pypy__.strategy(d) == "UnicodeDictStrategy" + assert len(d) == 2 + assert d[u"a"] == 5 + assert d[u"b"] == u"x" + assert u"c" not in d + + def test_setdefault(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert d.setdefault(u"a", "blub") == 1 + d.setdefault(u"x", 23) + assert __pypy__.strategy(d) == "UnicodeDictStrategy" + assert len(d) == 3 + assert d == {u"a": 1, u"b": "x", u"x": 23} + + def test_delitem(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + del d[u"a"] + assert __pypy__.strategy(d) == "UnicodeDictStrategy" + assert len(d) == 1 + assert d == {u"b": "x"} + + def test_popitem(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + k, v = d.popitem() + assert __pypy__.strategy(d) == "UnicodeDictStrategy" + if k == u"a": + assert v == 1 + assert len(d) == 1 + assert d == {u"b": "x"} + else: + assert v == u"x" + assert len(d) == 1 + assert d == {u"a": 1} + + def test_keys_value_items(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert d.keys() == [u"a", u"b"] + assert d.values() == [1, u"x"] + assert d.items() == [(u"a", 1), (u"b", u"x")] + + def test_iter_keys_value_items(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert list(d.iterkeys()) == [u"a", u"b"] + assert list(d.itervalues()) == [1, u"x"] + assert list(d.iteritems()) == [(u"a", 1), (u"b", u"x")] From pypy.commits at gmail.com Wed Jun 5 09:49:55 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 06:49:55 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: remove get_terminator Message-ID: <5cf7c883.1c69fb81.27526.c33e@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96757:5b4720a4e5fc Date: 2019-06-05 15:13 +0200 http://bitbucket.org/pypy/pypy/changeset/5b4720a4e5fc/ Log: remove get_terminator diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -610,7 +610,7 @@ self._raise("Key name must be string at char %d", i) i += 1 w_key = self._decode_key_string(i) - return currmap.get_next(w_key, self.s, start, self.pos) + return currmap.get_next(w_key, self.s, start, self.pos, self.startmap) def _decode_key_string(self, i): ll_chars = self.ll_chars @@ -732,12 +732,6 @@ self.instantiation_count = 0 self.number_of_leaves = 1 - def get_terminator(self): - while isinstance(self, JSONMap): - self = self.prev - assert isinstance(self, Terminator) - return self - def _check_invariants(self): if self.all_next: for next in self.all_next.itervalues(): @@ -745,7 +739,7 @@ elif self.single_nextmap: self.single_nextmap._check_invariants() - def get_next(self, w_key, string, start, stop): + def get_next(self, w_key, string, start, stop, terminator): from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq if isinstance(self, JSONMap): assert not self.state == MapBase.BLOCKED @@ -776,7 +770,6 @@ # fix number_of_leaves self.change_number_of_leaves(1) - terminator = self.get_terminator() terminator.register_potential_fringe(next) return next @@ -804,7 +797,7 @@ self.instantiation_count += 1 if isinstance(self, JSONMap) and self.state == MapBase.FRINGE: if self.is_useful(): - self.mark_useful() + self.mark_useful(self.startmap) def _make_next_map(self, w_key, key_repr): return JSONMap(self.space, self, w_key, key_repr) @@ -842,23 +835,6 @@ p.write("\n".join(r)) graphclient.display_dot_file(str(p)) - def _get_caching_stats(self): - caching = 0 - num_maps = 1 - if isinstance(self, JSONMap) and self.should_cache() and self.decoded_strings > 200: - caching += 1 - - if self.all_next: - children = self.all_next.values() - elif self.single_nextmap: - children = [self.single_nextmap] - else: - children = [] - for child in children: - a, b = child._get_caching_stats() - caching += a - num_maps += b - return caching, num_maps class Terminator(MapBase): def __init__(self, space): @@ -891,7 +867,7 @@ del self.current_fringe[f] return assert min_fringe - min_fringe.mark_blocked() + min_fringe.mark_blocked(self) del self.current_fringe[min_fringe] @@ -953,7 +929,7 @@ MapBase._check_invariants(self) - def mark_useful(self): + def mark_useful(self, terminator): # mark self as useful, and also the most commonly instantiated # children, recursively assert self.state in (MapBase.FRINGE, MapBase.PRELIMINARY) @@ -964,21 +940,20 @@ if child.instantiation_count > maxchild.instantiation_count: maxchild = child if maxchild is not None: - maxchild.mark_useful() + maxchild.mark_useful(terminator) if self.all_next: - terminator = self.get_terminator() for child in self.all_next.itervalues(): if child is not maxchild: terminator.register_potential_fringe(child) self.single_nextmap = maxchild - def mark_blocked(self): + def mark_blocked(self, terminator): self.state = MapBase.BLOCKED if self.all_next: for next in self.all_next.itervalues(): - next.mark_blocked() + next.mark_blocked(terminator) elif self.single_nextmap: - self.single_nextmap.mark_blocked() + self.single_nextmap.mark_blocked(terminator) self.single_nextmap = None self.all_next = None self.change_number_of_leaves(-self.number_of_leaves + 1) diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -19,7 +19,7 @@ w_a = self.space.newutf8("a", 1) w_b = self.space.newutf8("b", 1) w_c = self.space.newutf8("c", 1) - m1 = m.get_next(w_a, '"a"', 0, 3) + m1 = m.get_next(w_a, '"a"', 0, 3, m) assert m1.w_key == w_a assert m1.single_nextmap is None assert m1.key_repr == '"a"' @@ -27,16 +27,16 @@ assert not m1.key_repr_cmp('b": 123', 0) assert m.single_nextmap.w_key == w_a - m2 = m.get_next(w_a, '"a"', 0, 3) + m2 = m.get_next(w_a, '"a"', 0, 3, m) assert m2 is m1 - m3 = m.get_next(w_b, '"b"', 0, 3) + m3 = m.get_next(w_b, '"b"', 0, 3, m) assert m3.w_key == w_b assert m3.single_nextmap is None assert m3.key_repr == '"b"' assert m.single_nextmap is m1 - m4 = m3.get_next(w_c, '"c"', 0, 3) + m4 = m3.get_next(w_c, '"c"', 0, 3, m) assert m4.w_key == w_c assert m4.single_nextmap is None assert m4.key_repr == '"c"' @@ -47,15 +47,15 @@ w_a = self.space.newutf8("a", 1) w_b = self.space.newutf8("b", 1) w_c = self.space.newutf8("c", 1) - m1 = m.get_next(w_a, 'a"', 0, 2) + m1 = m.get_next(w_a, 'a"', 0, 2, m) assert m1.get_index(w_a) == 0 assert m1.get_index(w_b) == -1 - m2 = m.get_next(w_b, 'b"', 0, 2) + m2 = m.get_next(w_b, 'b"', 0, 2, m) assert m2.get_index(w_b) == 0 assert m2.get_index(w_a) == -1 - m3 = m2.get_next(w_c, 'c"', 0, 2) + m3 = m2.get_next(w_c, 'c"', 0, 2, m) assert m3.get_index(w_b) == 0 assert m3.get_index(w_c) == 1 assert m3.get_index(w_a) == -1 @@ -103,10 +103,10 @@ w_d = self.space.newutf8("d", 1) base = Terminator(self.space) base.instantiation_count = 6 - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = m1.get_next(w_b, 'b"', 0, 2) - m3 = m2.get_next(w_c, 'c"', 0, 2) - m4 = m2.get_next(w_d, 'd"', 0, 2) + m1 = base.get_next(w_a, 'a"', 0, 2, base) + m2 = m1.get_next(w_b, 'b"', 0, 2, base) + m3 = m2.get_next(w_c, 'c"', 0, 2, base) + m4 = m2.get_next(w_d, 'd"', 0, 2, base) return base, m1, m2, m3, m4 # unit tests for map state transistions @@ -126,7 +126,7 @@ assert m4.state == MapBase.PRELIMINARY m4.instantiation_count = 4 - m1.mark_useful() + m1.mark_useful(base) assert m1.state == MapBase.USEFUL assert m2.state == MapBase.USEFUL assert m3.state == MapBase.FRINGE @@ -144,7 +144,7 @@ assert m2.number_of_leaves == 2 assert m3.number_of_leaves == 1 assert m4.number_of_leaves == 1 - m5 = m2.get_next(w_x, 'x"', 0, 2) + m5 = m2.get_next(w_x, 'x"', 0, 2, base) assert base.number_of_leaves == 3 assert m1.number_of_leaves == 3 assert m2.number_of_leaves == 3 @@ -160,7 +160,7 @@ m4.instantiation_count = 4 assert base.current_fringe == {m1: None} - m1.mark_useful() + m1.mark_useful(base) assert base.current_fringe == {m1: None, m3: None} # not cleaned up base.cleanup_fringe() assert base.current_fringe == {m3: None} @@ -172,11 +172,11 @@ w_d = self.space.newutf8("d", 1) base = Terminator(self.space) base.instantiation_count = 6 - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = base.get_next(w_b, 'b"', 0, 2) - m3 = base.get_next(w_c, 'c"', 0, 2) - m4 = base.get_next(w_d, 'd"', 0, 2) - m5 = m4.get_next(w_a, 'a"', 0, 2) + m1 = base.get_next(w_a, 'a"', 0, 2, base) + m2 = base.get_next(w_b, 'b"', 0, 2, base) + m3 = base.get_next(w_c, 'c"', 0, 2, base) + m4 = base.get_next(w_d, 'd"', 0, 2, base) + m5 = m4.get_next(w_a, 'a"', 0, 2, base) base.instantiation_count = 7 m1.instantiation_count = 2 m2.instantiation_count = 2 @@ -202,9 +202,9 @@ s = '{"a": 1, "b": 2, "c": 3}' dec = JSONDecoder(space, s) dec.startmap = base = Terminator(space) - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = m1.get_next(w_b, 'b"', 0, 2) - m2.mark_blocked() + m1 = base.get_next(w_a, 'a"', 0, 2, base) + m2 = m1.get_next(w_b, 'b"', 0, 2, base) + m2.mark_blocked(base) w_res = dec.decode_object(1) assert space.int_w(space.len(w_res)) == 3 assert space.int_w(space.getitem(w_res, w_a)) == 1 @@ -219,35 +219,14 @@ w_u = self.space.newutf8("u", 1) space = self.space base = Terminator(space) - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = m1.get_next(w_b, 'b"', 0, 2) - m2.get_next(w_x, 'x"', 0, 2) - m2.get_next(w_u, 'u"', 0, 2) + m1 = base.get_next(w_a, 'a"', 0, 2, base) + m2 = m1.get_next(w_b, 'b"', 0, 2, base) + m2.get_next(w_x, 'x"', 0, 2, base) + m2.get_next(w_u, 'u"', 0, 2, base) assert base.number_of_leaves == 2 - m2.mark_blocked() + m2.mark_blocked(base) assert base.number_of_leaves == 1 - @pytest.mark.skip() - def test_caching_stats(self): - w_a = self.space.newutf8("a", 1) - w_b = self.space.newutf8("b", 1) - w_x = self.space.newutf8("x", 1) - w_u = self.space.newutf8("u", 1) - space = self.space - base = Terminator(space) - m1 = base.get_next(w_a, 'a"', 0, 2) - m2 = m1.get_next(w_b, 'b"', 0, 2) - m2.get_next(w_x, 'x"', 0, 2) - m2.get_next(w_u, 'u"', 0, 2) - m1.decode_string = 300 - m1.cache_hits = 0 - m3 = base.get_next(w_b, '"b"', 0, 3) - m3.decode_string = 300 - m3.cache_hits = 300 - caching_maps, total_maps = base._get_caching_stats() - assert caching_maps == 5 - assert total_maps == 6 - class AppTest(object): spaceconfig = {"objspace.usemodules._pypyjson": True} From pypy.commits at gmail.com Wed Jun 5 09:49:57 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 06:49:57 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: simplify fringe handling Message-ID: <5cf7c885.1c69fb81.a3f54.9e87@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96758:5dc3d4fba79a Date: 2019-06-05 15:27 +0200 http://bitbucket.org/pypy/pypy/changeset/5dc3d4fba79a/ Log: simplify fringe handling diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -595,7 +595,7 @@ def decode_key(self, i, currmap): newmap = self._decode_key(i, currmap) - currmap.observe_transition(newmap) + currmap.observe_transition(newmap, self.startmap) return newmap def _decode_key(self, i, currmap): @@ -789,7 +789,7 @@ decoder.pos = position + len(single_nextmap.key_repr) return single_nextmap - def observe_transition(self, newmap): + def observe_transition(self, newmap, terminator): """ observe a transition from self to newmap. This does a few things, including updating the self size estimate with the knowledge that one object transitioned from self to newmap. @@ -797,7 +797,7 @@ self.instantiation_count += 1 if isinstance(self, JSONMap) and self.state == MapBase.FRINGE: if self.is_useful(): - self.mark_useful(self.startmap) + self.mark_useful(terminator) def _make_next_map(self, w_key, key_repr): return JSONMap(self.space, self, w_key, key_repr) @@ -852,23 +852,21 @@ self.cleanup_fringe() self.current_fringe[prelim] = None + def remove_from_fringe(self, former_fringe): + assert former_fringe.state in (MapBase.USEFUL, MapBase.BLOCKED) + del self.current_fringe[former_fringe] + def cleanup_fringe(self): min_fringe = None min_avg = 10000000000 for f in self.current_fringe: - if f.state == MapBase.FRINGE: - avg = f.average_instantiation() - if avg < min_avg: - min_avg = avg - min_fringe = f - else: - for f in self.current_fringe.keys(): - if f.state != MapBase.FRINGE: - del self.current_fringe[f] - return + assert f.state == MapBase.FRINGE + avg = f.average_instantiation() + if avg < min_avg: + min_avg = avg + min_fringe = f assert min_fringe min_fringe.mark_blocked(self) - del self.current_fringe[min_fringe] class JSONMap(MapBase): @@ -932,8 +930,11 @@ def mark_useful(self, terminator): # mark self as useful, and also the most commonly instantiated # children, recursively + was_fringe = self.state == MapBase.FRINGE assert self.state in (MapBase.FRINGE, MapBase.PRELIMINARY) self.state = MapBase.USEFUL + if was_fringe: + terminator.remove_from_fringe(self) maxchild = self.single_nextmap if self.all_next is not None: for child in self.all_next.itervalues(): @@ -948,7 +949,10 @@ self.single_nextmap = maxchild def mark_blocked(self, terminator): + was_fringe = self.state == MapBase.FRINGE self.state = MapBase.BLOCKED + if was_fringe: + terminator.remove_from_fringe(self) if self.all_next: for next in self.all_next.itervalues(): next.mark_blocked(terminator) diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -150,7 +150,7 @@ assert m2.number_of_leaves == 3 assert m5.number_of_leaves == 1 - def test_cleanup_fringe_simple(self): + def test_mark_useful_cleans_fringe(self): base, m1, m2, m3, m4 = self._make_some_maps() base.instantiation_count = 6 assert m1.state == MapBase.FRINGE @@ -161,11 +161,9 @@ assert base.current_fringe == {m1: None} m1.mark_useful(base) - assert base.current_fringe == {m1: None, m3: None} # not cleaned up - base.cleanup_fringe() assert base.current_fringe == {m3: None} - def test_cleanup_fringe_block(self): + def test_cleanup_fringe(self): w_a = self.space.newutf8("a", 1) w_b = self.space.newutf8("b", 1) w_c = self.space.newutf8("c", 1) From pypy.commits at gmail.com Wed Jun 5 09:49:59 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 06:49:59 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: add some comments, some cleanups Message-ID: <5cf7c887.1c69fb81.a21a6.5fa7@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96759:a809240c1f45 Date: 2019-06-05 15:48 +0200 http://bitbucket.org/pypy/pypy/changeset/a809240c1f45/ Log: add some comments, some cleanups diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -91,6 +91,10 @@ self.lru_index = 0 self.startmap = self.space.fromcache(Terminator) + + # keep a list of objects that are created with maps that aren't clearly + # useful. If they turn out to be useful in the end we are good, + # otherwise convert them to dicts (see .close()) self.unclear_objects = [] self.scratch = [[None] * self.DEFAULT_SIZE_SCRATCH] # list of scratch space @@ -102,7 +106,7 @@ # clean up objects that are instances of now blocked maps for w_obj in self.unclear_objects: jsonmap = self._get_jsonmap_from_dict(w_obj) - if jsonmap.is_blocked(): + if jsonmap.is_state_blocked(): self._devolve_jsonmap_dict(w_obj) def getslice(self, start, end): @@ -353,21 +357,20 @@ i += 1 if ch == '}': self.pos = i - if currmap.is_blocked(): + self.scratch.append(values_w) # can reuse next time + if currmap.is_state_blocked(): currmap.instantiation_count += 1 - self.scratch.append(values_w) # can reuse next time dict_w = self._switch_to_dict(currmap, values_w, nextindex) return self._create_dict(dict_w) - self.scratch.append(values_w) # can reuse next time values_w = values_w[:nextindex] currmap.instantiation_count += 1 w_res = self._create_dict_map(values_w, currmap) - if currmap.state != MapBase.USEFUL: + if not currmap.is_state_useful(): self.unclear_objects.append(w_res) return w_res elif ch == ',': i = self.skip_whitespace(i) - if currmap.is_blocked(): + if currmap.is_state_blocked(): currmap.instantiation_count += 1 self.scratch.append(values_w) # can reuse next time dict_w = self._switch_to_dict(currmap, values_w, nextindex) @@ -660,6 +663,9 @@ class MapBase(object): + """ A map implementation to speed up parsing of json dicts, and to + represent the resulting dicts more compactly and make access faster. """ + # the basic problem we are trying to solve is the following: dicts in # json can either be used as objects, or as dictionaries with arbitrary # string keys. We want to use maps for the former, but not for the @@ -674,6 +680,16 @@ # into a "blocked" map, which is a point in the map tree where we will # switch to regular dicts, when we reach that part of the tree. + # One added complication: We want to keep the number of preliminary maps + # bounded to prevent generating tons of useless maps. but also not too + # small, to support having a json file that contains many uniform objects + # with tons of keys. That's where the idea of "fringe" maps comes into + # play. They are maps that sit between known useful nodes and preliminary + # nodes in the map transition tree. We bound only the number of fringe + # nodes we are considering (to MAX_FRINGE), but not the number of + # preliminary maps. When we have too many fringe maps, we remove the least + # commonly instantiated fringe map and mark it as blocked. + # allowed graph edges or nodes in all_next: # USEFUL ------- # / \ \ @@ -837,15 +853,19 @@ class Terminator(MapBase): + """ The root node of the map transition tree. """ def __init__(self, space): MapBase.__init__(self, space) - self.all_object_count = 0 + # a set of all map nodes that are currently in the FRINGE state self.current_fringe = {} def register_potential_fringe(self, prelim): + """ add prelim to the fringe, if its prev is either a Terminator or + useful. """ prev = prelim.prev if (isinstance(prev, Terminator) or isinstance(prev, JSONMap) and prev.state == MapBase.USEFUL): + assert prelim.state == MapBase.PRELIMINARY prelim.state = MapBase.FRINGE if len(self.current_fringe) > MapBase.MAX_FRINGE: @@ -853,10 +873,12 @@ self.current_fringe[prelim] = None def remove_from_fringe(self, former_fringe): + """ Remove former_fringe from self.current_fringe. """ assert former_fringe.state in (MapBase.USEFUL, MapBase.BLOCKED) del self.current_fringe[former_fringe] def cleanup_fringe(self): + """ remove the least-instantiated fringe map and block it.""" min_fringe = None min_avg = 10000000000 for f in self.current_fringe: @@ -868,6 +890,9 @@ assert min_fringe min_fringe.mark_blocked(self) + def _check_invariants(self): + for fringe in self.current_fringe: + assert fringe.state == MapBase.FRINGE class JSONMap(MapBase): """ A map implementation to speed up parsing """ @@ -890,8 +915,7 @@ self.keys_in_order = None self.strategy_instance = None - @jit.elidable - def get_terminator(self): + def _get_terminator(self): # only for _check_invariants while isinstance(self, JSONMap): self = self.prev assert isinstance(self, Terminator) @@ -924,6 +948,8 @@ if self.state == MapBase.BLOCKED: assert self.single_nextmap is None assert self.all_next is None + elif self.state == MapBase.FRINGE: + assert self in self._get_terminator().current_fringe MapBase._check_invariants(self) @@ -962,9 +988,12 @@ self.all_next = None self.change_number_of_leaves(-self.number_of_leaves + 1) - def is_blocked(self): + def is_state_blocked(self): return self.state == MapBase.BLOCKED + def is_state_useful(self): + return self.state == MapBase.USEFUL + def average_instantiation(self): return self.instantiation_count / float(self.number_of_leaves) From pypy.commits at gmail.com Wed Jun 5 16:00:01 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 13:00:01 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: cleanup instantiation_count and number_of_leaves, some comments, some renaming Message-ID: <5cf81f41.1c69fb81.a94ee.5dd3@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96760:ec6df04c89ba Date: 2019-06-05 18:49 +0200 http://bitbucket.org/pypy/pypy/changeset/ec6df04c89ba/ Log: cleanup instantiation_count and number_of_leaves, some comments, some renaming diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -24,17 +24,6 @@ return 0.0 return x * NEG_POW_10[exp] -def _compare_cache_entry(space, res, ll_chars, start, length): - if length != len(res): - return False - index = start - for c in res: - x = ord(c) - if not ll_chars[index] == chr(x): - return False - index += 1 - return True - class IntCache(object): START = -10 @@ -84,6 +73,9 @@ self.pos = 0 self.intcache = space.fromcache(IntCache) + # two caches, one for keys, one for general strings. they both have the + # form {hash-as-int: CacheEntry} and they don't deal with + # collisions at all. For every hash there is simply one string stored. self.cache = {} self.cache_wrapped = {} @@ -339,7 +331,7 @@ currmap = self.startmap while True: # parse a key: value - currmap = self.decode_key(i, currmap) + currmap = self.decode_key_map(i, currmap) i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] if ch != ':': @@ -359,11 +351,9 @@ self.pos = i self.scratch.append(values_w) # can reuse next time if currmap.is_state_blocked(): - currmap.instantiation_count += 1 dict_w = self._switch_to_dict(currmap, values_w, nextindex) return self._create_dict(dict_w) values_w = values_w[:nextindex] - currmap.instantiation_count += 1 w_res = self._create_dict_map(values_w, currmap) if not currmap.is_state_useful(): self.unclear_objects.append(w_res) @@ -371,7 +361,6 @@ elif ch == ',': i = self.skip_whitespace(i) if currmap.is_state_blocked(): - currmap.instantiation_count += 1 self.scratch.append(values_w) # can reuse next time dict_w = self._switch_to_dict(currmap, values_w, nextindex) return self.decode_object_dict(i, start, dict_w) @@ -582,27 +571,30 @@ if ((context is not None and context.decoded_strings < self.STRING_CACHE_EVALUATION_SIZE) or strhash in self.lru_cache): - entry = WrappedCacheEntry( + entry = CacheEntry( self.getslice(start, start + length), w_res) self.cache_wrapped[strhash] = entry else: self.lru_cache[self.lru_index] = strhash self.lru_index = (self.lru_index + 1) & self.LRU_MASK return w_res - if not _compare_cache_entry(self.space, entry.repr, ll_chars, start, length): - # hopefully rare + if not entry.compare(ll_chars, start, length): + # collision! hopefully rare return self._create_string_wrapped(start, i, nonascii) if context is not None: context.cache_hits += 1 return entry.w_uni - def decode_key(self, i, currmap): - newmap = self._decode_key(i, currmap) + def decode_key_map(self, i, currmap): + """ Given the current map currmap of an object, decode the next key at + position i. This returns the new map of the object. """ + newmap = self._decode_key_map(i, currmap) currmap.observe_transition(newmap, self.startmap) return newmap - def _decode_key(self, i, currmap): + def _decode_key_map(self, i, currmap): ll_chars = self.ll_chars + # first try to see whether we happen to find currmap.single_nextmap nextmap = currmap.fast_path_key_parse(self, i) if nextmap is not None: return nextmap @@ -616,6 +608,8 @@ return currmap.get_next(w_key, self.s, start, self.pos, self.startmap) def _decode_key_string(self, i): + """ decode key at position i as a string. Key strings are always + cached, since they repeat a lot. """ ll_chars = self.ll_chars start = i @@ -636,14 +630,13 @@ entry = self.cache[strhash] except KeyError: w_res = self._create_string_wrapped(start, i, nonascii) - entry = WrappedCacheEntry( + entry = CacheEntry( self.getslice(start, start + length), w_res) self.cache[strhash] = entry return w_res - if not _compare_cache_entry(self.space, entry.repr, ll_chars, start, length): - # hopefully rare + if not entry.compare(ll_chars, start, length): + # collision! hopefully rare w_res = self._create_string_wrapped(start, i, nonascii) - print w_res else: w_res = entry.w_uni return w_res @@ -656,11 +649,26 @@ i += 1 return self._decode_key_string(i) -class WrappedCacheEntry(object): +class CacheEntry(object): + """ A cache entry, bunding the encoded version of a string, and its wrapped + decoded variant. """ def __init__(self, repr, w_uni): + # repr is the escaped string self.repr = repr + # uni is the wrapped decoded string self.w_uni = w_uni + def compare(self, ll_chars, start, length): + """ Check whether self.repr occurs at ll_chars[start:start+length] """ + if length != len(self.repr): + return False + index = start + for c in self.repr: + if not ll_chars[index] == c: + return False + index += 1 + return True + class MapBase(object): """ A map implementation to speed up parsing of json dicts, and to @@ -745,6 +753,9 @@ # but then it also contains .single_nextmap self.all_next = None # later dict {key: nextmap} + # keep some statistics about every map: how often it was instantiated + # and how many non-blocked leaves the map transition tree has, starting + # from self self.instantiation_count = 0 self.number_of_leaves = 1 @@ -783,13 +794,15 @@ next = self._make_next_map(w_key, string[start:stop]) self.all_next[w_key] = next - # fix number_of_leaves + # one new leaf has been created self.change_number_of_leaves(1) terminator.register_potential_fringe(next) return next def change_number_of_leaves(self, difference): + if not difference: + return parent = self while isinstance(parent, JSONMap): parent.number_of_leaves += difference @@ -797,6 +810,9 @@ parent.number_of_leaves += difference # terminator def fast_path_key_parse(self, decoder, position): + """ Fast path when parsing the next key: We speculate that we will + always see a commonly seen next key, and use strcmp (implemented in + key_repr_cmp) to check whether that is the case. """ single_nextmap = self.single_nextmap if single_nextmap: ll_chars = decoder.ll_chars @@ -810,7 +826,7 @@ This does a few things, including updating the self size estimate with the knowledge that one object transitioned from self to newmap. also it potentially decides that self should move to state USEFUL.""" - self.instantiation_count += 1 + newmap.instantiation_count += 1 if isinstance(self, JSONMap) and self.state == MapBase.FRINGE: if self.is_useful(): self.mark_useful(terminator) @@ -915,6 +931,10 @@ self.keys_in_order = None self.strategy_instance = None + def __repr__(self): + return "" % ( + self.key_repr, self.instantiation_count, self.number_of_leaves, self.prev) + def _get_terminator(self): # only for _check_invariants while isinstance(self, JSONMap): self = self.prev @@ -961,6 +981,8 @@ self.state = MapBase.USEFUL if was_fringe: terminator.remove_from_fringe(self) + # find the most commonly instantiated child, store it into + # single_nextmap and mark it useful, recursively maxchild = self.single_nextmap if self.all_next is not None: for child in self.all_next.itervalues(): @@ -975,6 +997,7 @@ self.single_nextmap = maxchild def mark_blocked(self, terminator): + """ mark self and recursively all its children as blocked.""" was_fringe = self.state == MapBase.FRINGE self.state = MapBase.BLOCKED if was_fringe: @@ -1059,7 +1082,7 @@ if self.all_next is None: l = int(self.single_nextmap is not None) else: - l = len(self.all_next) + 1 + l = len(self.all_next) extra = "" if self.decoded_strings: extra = "\\n%s/%s (%s%%)" % (self.cache_hits, self.decoded_strings, self.cache_hits/float(self.decoded_strings)) diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -60,23 +60,23 @@ assert m3.get_index(w_c) == 1 assert m3.get_index(w_a) == -1 - def test_decode_key(self): + def test_decode_key_map(self): m = Terminator(self.space) m_diff = Terminator(self.space) for s1 in ["abc", "1001" * 10, u"ä".encode("utf-8")]: s = ' "%s" "%s" "%s"' % (s1, s1, s1) dec = JSONDecoder(self.space, s) assert dec.pos == 0 - m1 = dec.decode_key(dec.skip_whitespace(0), m) + m1 = dec.decode_key_map(dec.skip_whitespace(0), m) assert m1.w_key._utf8 == s1 assert m1.key_repr == '"%s"' % s1 # check caching on w_key level - m2 = dec.decode_key(dec.skip_whitespace(dec.pos), m_diff) + m2 = dec.decode_key_map(dec.skip_whitespace(dec.pos), m_diff) assert m1.w_key is m2.w_key # check caching on map level - m3 = dec.decode_key(dec.skip_whitespace(dec.pos), m_diff) + m3 = dec.decode_key_map(dec.skip_whitespace(dec.pos), m_diff) assert m3 is m2 dec.close() @@ -150,6 +150,14 @@ assert m2.number_of_leaves == 3 assert m5.number_of_leaves == 1 + def test_number_of_leaves_after_mark_blocked(self): + w_x = self.space.newutf8("x", 1) + base, m1, m2, m3, m4 = self._make_some_maps() + m5 = m2.get_next(w_x, 'x"', 0, 2, base) + assert base.number_of_leaves == 3 + m2.mark_blocked(base) + assert base.number_of_leaves == 1 + def test_mark_useful_cleans_fringe(self): base, m1, m2, m3, m4 = self._make_some_maps() base.instantiation_count = 6 @@ -225,6 +233,19 @@ m2.mark_blocked(base) assert base.number_of_leaves == 1 + def test_instatiation_count(self): + m = Terminator(self.space) + dec = JSONDecoder(self.space, '"abc" "def"') + m1 = dec.decode_key_map(dec.skip_whitespace(0), m) + m2 = dec.decode_key_map(dec.skip_whitespace(6), m1) + m1 = dec.decode_key_map(dec.skip_whitespace(0), m) + m2 = dec.decode_key_map(dec.skip_whitespace(6), m1) + m1 = dec.decode_key_map(dec.skip_whitespace(0), m) + + assert m1.instantiation_count == 3 + assert m2.instantiation_count == 2 + dec.close() + class AppTest(object): spaceconfig = {"objspace.usemodules._pypyjson": True} From pypy.commits at gmail.com Wed Jun 5 16:00:03 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 13:00:03 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: more commonts, some simplifications Message-ID: <5cf81f43.1c69fb81.67729.66da@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96761:7876645a7f18 Date: 2019-06-05 21:54 +0200 http://bitbucket.org/pypy/pypy/changeset/7876645a7f18/ Log: more commonts, some simplifications diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -26,6 +26,8 @@ class IntCache(object): + """ A cache for wrapped ints between START and END """ + START = -10 END = 256 @@ -89,7 +91,9 @@ # otherwise convert them to dicts (see .close()) self.unclear_objects = [] - self.scratch = [[None] * self.DEFAULT_SIZE_SCRATCH] # list of scratch space + # this is a freelist of lists that store the decoded value of an + # object, before they get copied into the eventual dict + self.scratch = [[None] * self.DEFAULT_SIZE_SCRATCH] def close(self): @@ -116,11 +120,13 @@ break return i - def decode_any(self, i): + def decode_any(self, i, contextmap=None): + """ Decode an object at position i. Optionally pass a contextmap, if + the value is decoded as the value of a dict. """ i = self.skip_whitespace(i) ch = self.ll_chars[i] if ch == '"': - return self.decode_string(i+1) + return self.decode_string(i+1, contextmap) elif ch == '[': return self.decode_array(i+1) elif ch == '{': @@ -276,13 +282,8 @@ self._raise("Unexpected '%s' when decoding object (char %d)", ch, i) - def decode_surrogate_pair(self, i, highsurr): - """ uppon enter the following must hold: - chars[i] == "\\" and chars[i+1] == "u" - """ - # the possible ValueError is caught by the caller - def decode_array(self, i): + """ Decode a list. i must be after the opening '[' """ w_list = self.space.newlist([]) start = i i = self.skip_whitespace(start) @@ -308,13 +309,6 @@ self._raise("Unexpected '%s' when decoding array (char %d)", ch, i-1) - def decode_any_context(self, i, context): - i = self.skip_whitespace(i) - ch = self.ll_chars[i] - if ch == '"': - return self.decode_string(i+1, context) - return self.decode_any(i) - def decode_object(self, i): start = i @@ -338,7 +332,7 @@ self._raise("No ':' found at char %d", i) i += 1 - w_value = self.decode_any_context(i, currmap) + w_value = self.decode_any(i, currmap) if nextindex == len(values_w): # full values_w = values_w + [None] * len(values_w) # double @@ -527,7 +521,10 @@ return i - def decode_string(self, i, context=None): + def decode_string(self, i, contextmap=None): + """ Decode a string at position i (which is right after the opening "). + Optionally pass a contextmap, if the value is decoded as the value of a + dict. """ ll_chars = self.ll_chars start = i ch = ll_chars[i] @@ -536,9 +533,9 @@ return self.w_empty_string # surprisingly common cache = True - if context is not None: - context.decoded_strings += 1 - if not context.should_cache(): + if contextmap is not None: + contextmap.decoded_strings += 1 + if not contextmap.should_cache_strings(): cache = False if len(self.s) < self.MIN_SIZE_FOR_STRING_CACHE: cache = False @@ -568,8 +565,8 @@ w_res = self._create_string_wrapped(start, i, nonascii) # only add *some* strings to the cache, because keeping them all is # way too expensive - if ((context is not None and - context.decoded_strings < self.STRING_CACHE_EVALUATION_SIZE) or + if ((contextmap is not None and + contextmap.decoded_strings < self.STRING_CACHE_EVALUATION_SIZE) or strhash in self.lru_cache): entry = CacheEntry( self.getslice(start, start + length), w_res) @@ -581,8 +578,8 @@ if not entry.compare(ll_chars, start, length): # collision! hopefully rare return self._create_string_wrapped(start, i, nonascii) - if context is not None: - context.cache_hits += 1 + if contextmap is not None: + contextmap.cache_hits += 1 return entry.w_uni def decode_key_map(self, i, currmap): @@ -801,6 +798,7 @@ return next def change_number_of_leaves(self, difference): + """ add difference to .number_of_leaves of self and its parents """ if not difference: return parent = self @@ -974,8 +972,8 @@ MapBase._check_invariants(self) def mark_useful(self, terminator): - # mark self as useful, and also the most commonly instantiated - # children, recursively + """ mark self as useful, and also the most commonly instantiated + children, recursively """ was_fringe = self.state == MapBase.FRINGE assert self.state in (MapBase.FRINGE, MapBase.PRELIMINARY) self.state = MapBase.USEFUL @@ -1018,12 +1016,20 @@ return self.state == MapBase.USEFUL def average_instantiation(self): + """ the number of instantiations, divided by the number of leaves. we + want to favor nodes that have either a high instantiation count, or few + leaves below it. """ return self.instantiation_count / float(self.number_of_leaves) def is_useful(self): return self.average_instantiation() > self.USEFUL_THRESHOLD - def should_cache(self): + def should_cache_strings(self): + """ return whether strings parsed in the context of this map should be + cached. """ + # we should cache if either we've seen few strings so far (less than + # STRING_CACHE_EVALUATION_SIZE), or if we've seen many, and the cache + # hit rate has been high enough return not (self.decoded_strings > JSONDecoder.STRING_CACHE_EVALUATION_SIZE and self.cache_hits * JSONDecoder.STRING_CACHE_USEFULNESS_FACTOR < self.decoded_strings) From pypy.commits at gmail.com Wed Jun 5 16:01:43 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 05 Jun 2019 13:01:43 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: document branch Message-ID: <5cf81fa7.1c69fb81.72c32.a500@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96762:cb74a7af3b1d Date: 2019-06-05 22:00 +0200 http://bitbucket.org/pypy/pypy/changeset/cb74a7af3b1d/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -38,3 +38,8 @@ .. branch: optimizeopt-cleanup Cleanup optimizeopt + +.. branch: json-decoder-maps + +Much faster and more memory-efficient JSON decoding. The resulting +dictionaries that come out of the JSON decoder have faster lookups too. From pypy.commits at gmail.com Thu Jun 6 09:33:05 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 06 Jun 2019 06:33:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: trivial fixes Message-ID: <5cf91611.1c69fb81.e4d3d.b632@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96763:b0cb066e7e1f Date: 2019-06-06 16:32 +0300 http://bitbucket.org/pypy/pypy/changeset/b0cb066e7e1f/ Log: trivial fixes diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -793,7 +793,7 @@ finally: lltype.free(info_ptr, flavor='raw') return w_bytes - + @unwrap_spec(how="c_int") def shutdown_w(self, space, how): """shutdown(flag) diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -409,7 +409,7 @@ path = self.path3 with open(path, 'wb'): pass - os.unlink(self.Path()) + os.unlink(path) def test_times(self): """ diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py --- a/pypy/objspace/std/test/test_unicodeobject.py +++ b/pypy/objspace/std/test/test_unicodeobject.py @@ -728,7 +728,7 @@ raises(TypeError, 'hello'.translate) raises(ValueError, "\xff".translate, {0xff: sys.maxunicode+1}) - raises(TypeError, u'x'.translate, {ord('x'):0x110000}) + raises(ValueError, u'x'.translate, {ord('x'):0x110000}) def test_maketrans(self): assert 'abababc' == 'abababc'.translate({'b': ''}) From pypy.commits at gmail.com Thu Jun 6 10:48:09 2019 From: pypy.commits at gmail.com (arigo) Date: Thu, 06 Jun 2019 07:48:09 -0700 (PDT) Subject: [pypy-commit] pypy default: Issue #3014 Message-ID: <5cf927a9.1c69fb81.5d27.cc33@mx.google.com> Author: Armin Rigo Branch: Changeset: r96764:4570c3e9b030 Date: 2019-06-06 16:47 +0200 http://bitbucket.org/pypy/pypy/changeset/4570c3e9b030/ Log: Issue #3014 The gc_load family of operations must force the lazy setfields and setarrayitems to occur first diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -629,6 +629,20 @@ # and then emit the operation return self.emit(op) + def optimize_GC_LOAD_I(self, op): + # seeing a 'gc_load*' forces all the lazy sets that are still + # pending, as an approximation. We could try to be really clever + # and only force some of them, but we don't have any descr here. + self.force_all_lazy_sets() + self.make_nonnull(op.getarg(0)) + return self.emit(op) + optimize_GC_LOAD_R = optimize_GC_LOAD_I + optimize_GC_LOAD_F = optimize_GC_LOAD_I + + optimize_GC_LOAD_INDEXED_I = optimize_GC_LOAD_I + optimize_GC_LOAD_R = optimize_GC_LOAD_INDEXED_I + optimize_GC_LOAD_F = optimize_GC_LOAD_INDEXED_I + def optimize_QUASIIMMUT_FIELD(self, op): # Pattern: QUASIIMMUT_FIELD(s, descr=QuasiImmutDescr) # x = GETFIELD_GC(s, descr='inst_x') # pure diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -9495,3 +9495,40 @@ jump(i3, i2, i3) """ self.optimize_loop(ops, expected) + + def test_issue3014(self): + # 'gc_load_indexed' must force 'setarrayitem_gc' + ops = """ + [i183] + p0 = new_array(5, descr=arraydescr) + setarrayitem_gc(p0, 0, i183, descr=arraydescr) + i235 = gc_load_indexed_i(p0, 0, 1, 16, 2) + escape_i(i235) + jump(i183) + """ + self.optimize_loop(ops, ops) + + def test_issue3014_2(self): + # same rules for gc_store_indexed versus getarrayitem_gc + # (in this direction it seems to work already) + ops = """ + [i183] + p0 = new_array(5, descr=arraydescr) + gc_store_indexed(p0, 0, i183, 1, 16, 2) + i235 = getarrayitem_gc_i(p0, 0, descr=arraydescr) + escape_i(i235) + jump(i183) + """ + self.optimize_loop(ops, ops) + + def test_issue3014_3(self): + # 'gc_load' must force 'setfield_gc' + ops = """ + [i183] + p0 = new(descr=ssize) + setfield_gc(p0, i183, descr=adescr) + i235 = gc_load_i(p0, 8, 2) + escape_i(i235) + jump(i183) + """ + self.optimize_loop(ops, ops) From pypy.commits at gmail.com Thu Jun 6 10:50:48 2019 From: pypy.commits at gmail.com (arigo) Date: Thu, 06 Jun 2019 07:50:48 -0700 (PDT) Subject: [pypy-commit] pypy default: meh, fix for 4570c3e9b030 Message-ID: <5cf92848.1c69fb81.bfbd6.a457@mx.google.com> Author: Armin Rigo Branch: Changeset: r96765:7419bb5bbb41 Date: 2019-06-06 16:49 +0200 http://bitbucket.org/pypy/pypy/changeset/7419bb5bbb41/ Log: meh, fix for 4570c3e9b030 diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -640,8 +640,8 @@ optimize_GC_LOAD_F = optimize_GC_LOAD_I optimize_GC_LOAD_INDEXED_I = optimize_GC_LOAD_I - optimize_GC_LOAD_R = optimize_GC_LOAD_INDEXED_I - optimize_GC_LOAD_F = optimize_GC_LOAD_INDEXED_I + optimize_GC_LOAD_INDEXED_R = optimize_GC_LOAD_I + optimize_GC_LOAD_INDEXED_F = optimize_GC_LOAD_I def optimize_QUASIIMMUT_FIELD(self, op): # Pattern: QUASIIMMUT_FIELD(s, descr=QuasiImmutDescr) From pypy.commits at gmail.com Thu Jun 6 10:54:56 2019 From: pypy.commits at gmail.com (arigo) Date: Thu, 06 Jun 2019 07:54:56 -0700 (PDT) Subject: [pypy-commit] pypy default: Improve the test, still passing Message-ID: <5cf92940.1c69fb81.d6e2d.ce8e@mx.google.com> Author: Armin Rigo Branch: Changeset: r96766:631c5b49bd25 Date: 2019-06-06 16:54 +0200 http://bitbucket.org/pypy/pypy/changeset/631c5b49bd25/ Log: Improve the test, still passing diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -9509,15 +9509,17 @@ self.optimize_loop(ops, ops) def test_issue3014_2(self): - # same rules for gc_store_indexed versus getarrayitem_gc + # same rules for gc_store_indexed versus getarrayitem_gc, + # and 'gc_store_indexed' invalidates the value for 'getarrayitem_gc' # (in this direction it seems to work already) ops = """ - [i183] - p0 = new_array(5, descr=arraydescr) + [p0, i183] + i234 = getarrayitem_gc_i(p0, 0, descr=arraydescr) gc_store_indexed(p0, 0, i183, 1, 16, 2) i235 = getarrayitem_gc_i(p0, 0, descr=arraydescr) + escape_i(i234) escape_i(i235) - jump(i183) + jump(p0, i183) """ self.optimize_loop(ops, ops) From pypy.commits at gmail.com Fri Jun 7 08:44:52 2019 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 07 Jun 2019 05:44:52 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: typos Message-ID: <5cfa5c44.1c69fb81.d4419.a5ae@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96772:53dd110e73ce Date: 2019-06-06 08:25 +0200 http://bitbucket.org/pypy/pypy/changeset/53dd110e73ce/ Log: typos diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -475,7 +475,7 @@ elif ch == 'r': put('\r') elif ch == 't': put('\t') elif ch == 'u': - # may be a suggorate pair + # may be a surrogate pair return self.decode_escape_sequence_unicode(i, stringbuilder) else: self._raise("Invalid \\escape: %s (char %d)", ch, i-1) @@ -647,7 +647,7 @@ return self._decode_key_string(i) class CacheEntry(object): - """ A cache entry, bunding the encoded version of a string, and its wrapped + """ A cache entry, bundling the encoded version of a string, and its wrapped decoded variant. """ def __init__(self, repr, w_uni): # repr is the escaped string @@ -861,7 +861,7 @@ r = ["digraph G {"] self._all_dot(r) r.append("}") - p = pytest.ensuretemp("resilientast").join("temp.dot") + p = pytest.ensuretemp("jsonmap").join("temp.dot") p.write("\n".join(r)) graphclient.display_dot_file(str(p)) @@ -1016,7 +1016,7 @@ return self.state == MapBase.USEFUL def average_instantiation(self): - """ the number of instantiations, divided by the number of leaves. we + """ the number of instantiations, divided by the number of leaves. We want to favor nodes that have either a high instantiation count, or few leaves below it. """ return self.instantiation_count / float(self.number_of_leaves) From pypy.commits at gmail.com Fri Jun 7 08:44:54 2019 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 07 Jun 2019 05:44:54 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: fix translation maybe Message-ID: <5cfa5c46.1c69fb81.84076.d0c0@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96773:8455aa36a879 Date: 2019-06-07 10:10 +0200 http://bitbucket.org/pypy/pypy/changeset/8455aa36a879/ Log: fix translation maybe diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -894,7 +894,7 @@ def cleanup_fringe(self): """ remove the least-instantiated fringe map and block it.""" min_fringe = None - min_avg = 10000000000 + min_avg = float('inf') for f in self.current_fringe: assert f.state == MapBase.FRINGE avg = f.average_instantiation() From pypy.commits at gmail.com Fri Jun 7 09:09:55 2019 From: pypy.commits at gmail.com (Raemi) Date: Fri, 07 Jun 2019 06:09:55 -0700 (PDT) Subject: [pypy-commit] pypy guard-value-limit: add example for testing Message-ID: <5cfa6223.1c69fb81.84076.d990@mx.google.com> Author: Remi Meier Branch: guard-value-limit Changeset: r96774:c6bc0ea32200 Date: 2019-06-07 15:09 +0200 http://bitbucket.org/pypy/pypy/changeset/c6bc0ea32200/ Log: add example for testing diff --git a/pypy/goal/overspecialize_type.py b/pypy/goal/overspecialize_type.py new file mode 100644 --- /dev/null +++ b/pypy/goal/overspecialize_type.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +import sys + +def run(objs, n): + res = 0 + for o in objs: + for i in range(n): + res += o.f() + print res + +def thefun(): + return 1 + +if __name__ == "__main__": + try: + num_types = int(sys.argv[1]) + n = int(sys.argv[2]) + objs = [] + for i in range(num_types): + exec(""" +class A_%s(object): + field = %s + #def f(self): return 1#return self.field + """ % (i, i)) + objs.append(eval("A_%s()" % i)) + # use same 'thefun' for each obj (to avoid more failing guards) + setattr(objs[-1], 'f', thefun) + + run(objs, n) + except Exception as e: + print e + print "Arguments: num_types iterations" + print "env PYPYLOG=jit-sum:- ./pypy-c --jit guard_value_limit=8 overspecialize_type.py 42 50000" From pypy.commits at gmail.com Fri Jun 7 15:30:15 2019 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 07 Jun 2019 12:30:15 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: try again Message-ID: <5cfabb47.1c69fb81.5bcd0.0c60@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96776:89f77e4471e9 Date: 2019-06-07 21:29 +0200 http://bitbucket.org/pypy/pypy/changeset/89f77e4471e9/ Log: try again diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -894,7 +894,7 @@ def cleanup_fringe(self): """ remove the least-instantiated fringe map and block it.""" min_fringe = None - min_avg = float('inf') + min_avg = 1e200 for f in self.current_fringe: assert f.state == MapBase.FRINGE avg = f.average_instantiation() From pypy.commits at gmail.com Sat Jun 8 16:56:30 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sat, 08 Jun 2019 13:56:30 -0700 (PDT) Subject: [pypy-commit] pypy winconsoleio: Fixed a few translation issues Message-ID: <5cfc20fe.1c69fb81.24b6d.d9b3@mx.google.com> Author: andrewjlawrence Branch: winconsoleio Changeset: r96777:6c43ea44b9f0 Date: 2019-06-03 23:13 +0100 http://bitbucket.org/pypy/pypy/changeset/6c43ea44b9f0/ Log: Fixed a few translation issues diff --git a/pypy/module/_io/interp_win32consoleio.py b/pypy/module/_io/interp_win32consoleio.py --- a/pypy/module/_io/interp_win32consoleio.py +++ b/pypy/module/_io/interp_win32consoleio.py @@ -7,8 +7,9 @@ from pypy.module._io.interp_iobase import (W_RawIOBase, DEFAULT_BUFFER_SIZE) from pypy.interpreter.unicodehelper import fsdecode from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib._os_support import _preferred_traits, string_trait +from rpython.rlib._os_support import _preferred_traits from rpython.rlib import rwin32 +from rpython.rlib.rwin32file import make_win32_traits SMALLBUF = 4 @@ -55,7 +56,7 @@ m = 'r' elif normdecoded == unicodedata.normalize("NFKD", "CONOUT$".casefold()): m = 'w' - elif normaldecoded == unicodedata.normalize("NFKD", "CON".casefold()) + elif normaldecoded == unicodedata.normalize("NFKD", "CON".casefold()): m = 'x' if m != '\0': @@ -87,7 +88,7 @@ m = 'r' elif normdecoded == unicodedata.normalize("NFKD", "CONOUT$".casefold()): m = 'w' - elif normaldecoded == unicodedata.normalize("NFKD", "CON".casefold()) + elif normaldecoded == unicodedata.normalize("NFKD", "CON".casefold()): m = 'x' lltype.free(pname_buf, flavor='raw') @@ -112,6 +113,7 @@ def descr_init(self, space, w_nameobj, w_mode, w_closefd, w_opener): #self.fd = -1 #self.created = 0 + name = None self.readable = False self.writable = False #self.closehandle = 0; From pypy.commits at gmail.com Sat Jun 8 16:56:32 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sat, 08 Jun 2019 13:56:32 -0700 (PDT) Subject: [pypy-commit] pypy winconsoleio: Implemented _testconsole. Fixed translation issues. Message-ID: <5cfc2100.1c69fb81.e5974.4939@mx.google.com> Author: andrewjlawrence Branch: winconsoleio Changeset: r96778:fb78fa19bcc4 Date: 2019-06-08 21:55 +0100 http://bitbucket.org/pypy/pypy/changeset/fb78fa19bcc4/ Log: Implemented _testconsole. Fixed translation issues. diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -153,8 +153,6 @@ BOOL WINAPI GetQueuedCompletionStatus(HANDLE, LPDWORD, ULONG**, LPOVERLAPPED*, DWORD); HANDLE WINAPI CreateIoCompletionPort(HANDLE, HANDLE, ULONG_PTR, DWORD); - - #define WT_EXECUTEINWAITTHREAD 0x00000004 #define WT_EXECUTEONLYONCE 0x00000008 @@ -162,6 +160,55 @@ LPVOID HeapAlloc(HANDLE, DWORD, SIZE_T); BOOL HeapFree(HANDLE, DWORD, LPVOID); +typedef struct _COORD { + SHORT X; + SHORT Y; +} COORD, *PCOORD; + +typedef struct _FOCUS_EVENT_RECORD { + BOOL bSetFocus; +} FOCUS_EVENT_RECORD; + +typedef struct _WINDOW_BUFFER_SIZE_RECORD { + COORD dwSize; +} WINDOW_BUFFER_SIZE_RECORD; + +typedef struct _KEY_EVENT_RECORD { + BOOL bKeyDown; + WORD wRepeatCount; + WORD wVirtualKeyCode; + WORD wVirtualScanCode; + union { + WCHAR UnicodeChar; + CHAR AsciiChar; + } uChar; + DWORD dwControlKeyState; +} KEY_EVENT_RECORD; + +typedef struct _MENU_EVENT_RECORD { + UINT dwCommandId; +} MENU_EVENT_RECORD, *PMENU_EVENT_RECORD; + +typedef struct _MOUSE_EVENT_RECORD { + COORD dwMousePosition; + DWORD dwButtonState; + DWORD dwControlKeyState; + DWORD dwEventFlags; +} MOUSE_EVENT_RECORD; + +typedef struct _INPUT_RECORD { + WORD EventType; + union { + KEY_EVENT_RECORD KeyEvent; + MOUSE_EVENT_RECORD MouseEvent; + WINDOW_BUFFER_SIZE_RECORD WindowBufferSizeEvent; + MENU_EVENT_RECORD MenuEvent; + FOCUS_EVENT_RECORD FocusEvent; + } Event; +} INPUT_RECORD; + +BOOL WINAPI WriteConsoleInputW(HANDLE, const INPUT_RECORD*, DWORD, LPDWORD); + """) # -------------------- Win Sock 2 ---------------------- diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x05\x0D\x00\x01\x63\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x7F\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x59\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x03\x00\x01\x52\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x63\x03\x00\x00\x2B\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2B\x11\x00\x00\x2B\x11\x00\x01\x5E\x03\x00\x01\x53\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x03\x00\x00\x37\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x37\x11\x00\x00\x15\x11\x00\x01\x49\x03\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x25\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x19\x11\x00\x00\x07\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x5F\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x26\x11\x00\x01\x41\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x62\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x03\x00\x00\x26\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x67\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x37\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x7F\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xEE\x03\x00\x00\x07\x01\x00\x01\x62\x03\x00\x00\x19\x11\x00\x00\x05\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xBF\x11\x00\x00\xBF\x11\x00\x00\x33\x11\x00\x00\x34\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x74\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x0A\x01\x00\x00\x37\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x02\x0F\x00\x00\xE9\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x00\x0F\x00\x00\xE9\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x01\x58\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x01\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xFD\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xEE\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xBF\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x01\x06\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\xFD\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x01\x7F\x0D\x00\x00\x08\x01\x00\x00\x00\x0F\x00\x01\x7F\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x01\x7F\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x01\x7F\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x0C\x09\x00\x01\x4F\x03\x00\x00\x13\x09\x00\x01\x51\x03\x00\x00\x14\x09\x00\x00\x0D\x09\x00\x00\x09\x09\x00\x01\x55\x03\x00\x00\x0E\x09\x00\x01\x57\x03\x00\x00\x0A\x09\x00\x00\x0F\x09\x00\x00\x15\x09\x00\x01\x5D\x03\x00\x01\x5C\x03\x00\x00\x17\x09\x00\x00\x16\x09\x00\x00\x0B\x09\x00\x00\x10\x09\x00\x01\x61\x03\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x02\x01\x00\x01\x63\x05\x00\x00\x00\x0E\x00\x01\x63\x05\x00\x00\x00\x08\x00\x00\x51\x03\x00\x00\x57\x03\x00\x00\xA4\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x04\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x01\x76\x03\x00\x00\x04\x01\x00\x01\x76\x05\x00\x00\x00\x10\x00\x01\x76\x05\x00\x00\x00\x08\x00\x00\x02\x05\x00\x00\x00\x07\x00\x00\xE9\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xEE\x05\x00\x00\x01\x00', - _globals = (b'\x00\x00\x44\x23CancelIo',0,b'\x00\x00\x47\x23CancelIoEx',0,b'\x00\x00\x44\x23CloseHandle',0,b'\x00\x00\x47\x23ConnectNamedPipe',0,b'\x00\x00\xFC\x23CreateEventA',0,b'\x00\x01\x02\x23CreateEventW',0,b'\x00\x01\x08\x23CreateFileA',0,b'\x00\x01\x35\x23CreateFileW',0,b'\x00\x01\x23\x23CreateIoCompletionPort',0,b'\x00\x01\x11\x23CreateNamedPipeA',0,b'\x00\x01\x2B\x23CreateNamedPipeW',0,b'\x00\x00\x36\x23CreatePipe',0,b'\x00\x00\x2A\x23CreateProcessA',0,b'\x00\x00\xC5\x23CreateProcessW',0,b'\x00\x00\xAE\x23DuplicateHandle',0,b'\x00\x01\x3E\x23ExitProcess',0,b'\x00\x01\x29\x23GetCurrentProcess',0,b'\x00\x00\x76\x23GetExitCodeProcess',0,b'\x00\x00\xE6\x23GetLastError',0,b'\x00\x00\xE1\x23GetModuleFileNameW',0,b'\x00\x00\x4B\x23GetOverlappedResult',0,b'\x00\x00\xFA\x23GetProcessHeap',0,b'\x00\x00\x7A\x23GetQueuedCompletionStatus',0,b'\x00\x01\x20\x23GetStdHandle',0,b'\x00\x00\xE6\x23GetVersion',0,b'\x00\x00\xF5\x23HeapAlloc',0,b'\x00\x00\x1C\x23HeapFree',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\xF0\x23OpenProcess',0,b'\x00\x00\x9C\x23PeekNamedPipe',0,b'\x00\x00\x87\x23PostQueuedCompletionStatus',0,b'\x00\x00\x21\x23ReadFile',0,b'\x00\x00\x3C\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xD4\x23SetErrorMode',0,b'\x00\x00\x44\x23SetEvent',0,b'\x00\x01\x46\x23SetLastError',0,b'\x00\x00\x81\x23SetNamedPipeHandleState',0,b'\x00\x00\x72\x23TerminateProcess',0,b'\x00\x00\x44\x23UnregisterWait',0,b'\x00\x00\x98\x23UnregisterWaitEx',0,b'\x00\x00\x8D\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x60\x23WSARecv',0,b'\x00\x00\x69\x23WSASend',0,b'\x00\x00\xBE\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xD7\x23WaitForMultipleObjects',0,b'\x00\x00\xDD\x23WaitForSingleObject',0,b'\x00\x00\x00\x23WaitNamedPipeA',0,b'\x00\x00\xB7\x23WriteFile',0,b'\x00\x00\xD1\x23_get_osfhandle',0,b'\x00\x00\x28\x23_getch',0,b'\x00\x00\x28\x23_getche',0,b'\x00\x00\xEB\x23_getwch',0,b'\x00\x00\xEB\x23_getwche',0,b'\x00\x00\x28\x23_kbhit',0,b'\x00\x00\x0B\x23_locking',0,b'\x00\x00\x10\x23_open_osfhandle',0,b'\x00\x00\x04\x23_putch',0,b'\x00\x00\xED\x23_putwch',0,b'\x00\x00\x07\x23_setmode',0,b'\x00\x00\x04\x23_ungetch',0,b'\x00\x00\xE8\x23_ungetwch',0,b'\x00\x00\x17\x23bind',0,b'\x00\x00\x14\x23closesocket',0,b'\x00\x00\xE8\x23htons',0,b'\x00\x01\x1B\x23socket',0), - _struct_unions = ((b'\x00\x00\x01\x70\x00\x00\x00\x03$1',b'\x00\x01\x6C\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x01\x6C\x00\x00\x00\x02$2',b'\x00\x00\x02\x11Offset',b'\x00\x00\x02\x11OffsetHigh'),(b'\x00\x00\x01\x71\x00\x00\x00\x03$3',b'\x00\x01\x77\x11Byte',b'\x00\x01\x7D\x11Word'),(b'\x00\x00\x01\x72\x00\x00\x00\x01$4',b'\x00\x01\x6D\x11',b'\x00\x00\x02\x11Value'),(b'\x00\x00\x01\x6D\x00\x00\x00\x02$5',b'\x00\x00\x02\x13\x00\x00\x00\x1CZone',b'\x00\x00\x02\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x73\x00\x00\x00\x03$6',b'\x00\x00\x02\x11sin6_scope_id',b'\x00\x01\x57\x11sin6_scope_struct'),(b'\x00\x00\x01\x74\x00\x00\x00\x03$7',b'\x00\x01\x6E\x11S_un_b',b'\x00\x01\x6F\x11S_un_w',b'\x00\x00\x02\x11S_addr'),(b'\x00\x00\x01\x6E\x00\x00\x00\x02$8',b'\x00\x01\x76\x11s_b1',b'\x00\x01\x76\x11s_b2',b'\x00\x01\x76\x11s_b3',b'\x00\x01\x76\x11s_b4'),(b'\x00\x00\x01\x6F\x00\x00\x00\x02$9',b'\x00\x00\xE9\x11s_w1',b'\x00\x00\xE9\x11s_w2'),(b'\x00\x00\x01\x53\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x02\x11dwProcessId',b'\x00\x00\x02\x11dwThreadId'),(b'\x00\x00\x01\x57\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x72\x11'),(b'\x00\x00\x01\x5E\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x02\x11cb',b'\x00\x00\x2B\x11lpReserved',b'\x00\x00\x2B\x11lpDesktop',b'\x00\x00\x2B\x11lpTitle',b'\x00\x00\x02\x11dwX',b'\x00\x00\x02\x11dwY',b'\x00\x00\x02\x11dwXSize',b'\x00\x00\x02\x11dwYSize',b'\x00\x00\x02\x11dwXCountChars',b'\x00\x00\x02\x11dwYCountChars',b'\x00\x00\x02\x11dwFillAttribute',b'\x00\x00\x02\x11dwFlags',b'\x00\x00\xE9\x11wShowWindow',b'\x00\x00\xE9\x11cbReserved2',b'\x00\x01\x75\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x01\x4D\x00\x00\x00\x02_GUID',b'\x00\x00\x02\x11Data1',b'\x00\x00\xE9\x11Data2',b'\x00\x00\xE9\x11Data3',b'\x00\x01\x79\x11Data4'),(b'\x00\x00\x01\x52\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x02\x11Internal',b'\x00\x00\x02\x11InternalHigh',b'\x00\x01\x70\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x01\x55\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x15\x11hCompletionPort',b'\x00\x00\x26\x11Overlapped'),(b'\x00\x00\x01\x58\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x02\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x05\x11bInheritHandle'),(b'\x00\x00\x01\x5F\x00\x00\x00\x02_WSABUF',b'\x00\x00\x02\x11len',b'\x00\x00\x2B\x11buf'),(b'\x00\x00\x01\x61\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x05\x11ChainLen',b'\x00\x01\x7B\x11ChainEntries'),(b'\x00\x00\x01\x62\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x02\x11dwServiceFlags1',b'\x00\x00\x02\x11dwServiceFlags2',b'\x00\x00\x02\x11dwServiceFlags3',b'\x00\x00\x02\x11dwServiceFlags4',b'\x00\x00\x02\x11dwProviderFlags',b'\x00\x01\x4D\x11ProviderId',b'\x00\x00\x02\x11dwCatalogEntryId',b'\x00\x01\x61\x11ProtocolChain',b'\x00\x00\x05\x11iVersion',b'\x00\x00\x05\x11iAddressFamily',b'\x00\x00\x05\x11iMaxSockAddr',b'\x00\x00\x05\x11iMinSockAddr',b'\x00\x00\x05\x11iSocketType',b'\x00\x00\x05\x11iProtocol',b'\x00\x00\x05\x11iProtocolMaxOffset',b'\x00\x00\x05\x11iNetworkByteOrder',b'\x00\x00\x05\x11iSecurityScheme',b'\x00\x00\x02\x11dwMessageSize',b'\x00\x00\x02\x11dwProviderReserved',b'\x00\x01\x80\x11szProtocol'),(b'\x00\x00\x01\x4F\x00\x00\x00\x02in6_addr',b'\x00\x01\x71\x11u'),(b'\x00\x00\x01\x51\x00\x00\x00\x02in_addr',b'\x00\x01\x74\x11S_un'),(b'\x00\x00\x01\x59\x00\x00\x00\x02sockaddr',b'\x00\x00\xE9\x11sa_family',b'\x00\x01\x64\x11sa_data'),(b'\x00\x00\x01\x5D\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x6B\x11sin_family',b'\x00\x00\xE9\x11sin_port',b'\x00\x01\x51\x11sin_addr',b'\x00\x01\x66\x11sin_zero'),(b'\x00\x00\x01\x5C\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xE9\x11sin6_family',b'\x00\x00\xE9\x11sin6_port',b'\x00\x00\x02\x11sin6_flowinfo',b'\x00\x01\x4F\x11sin6_addr',b'\x00\x01\x73\x11')), - _typenames = (b'\x00\x00\x00\xE9ADDRESS_FAMILY',b'\x00\x00\x01\x6AAcceptExPtr',b'\x00\x00\x01\x69ConnectExPtr',b'\x00\x00\x01\x68DisconnectExPtr',b'\x00\x00\x01\x4DGUID',b'\x00\x00\x01\x4FIN6_ADDR',b'\x00\x00\x01\x51INADDR',b'\x00\x00\x01\x68LPFN_DISCONNECTEX',b'\x00\x00\x01\x4ELPIN6_ADDR',b'\x00\x00\x00\x26LPOVERLAPPED',b'\x00\x00\x00\x67LPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x34LPPROCESS_INFORMATION',b'\x00\x00\x01\x54LPPostCallbackData',b'\x00\x00\x00\xFDLPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19LPSOCKADDR',b'\x00\x00\x01\x5ALPSOCKADDR_IN',b'\x00\x00\x01\x5BLPSOCKADDR_IN6_LH',b'\x00\x00\x00\x33LPSTARTUPINFO',b'\x00\x00\x00\x62LPWSABUF',b'\x00\x00\x01\x60LPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xC1LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x52OVERLAPPED',b'\x00\x00\x01\x4EPIN6_ADDR',b'\x00\x00\x01\x50PINADDR',b'\x00\x00\x01\x53PROCESS_INFORMATION',b'\x00\x00\x01\x56PSCOPE_ID',b'\x00\x00\x00\xFDPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19PSOCKADDR',b'\x00\x00\x01\x5APSOCKADDR_IN',b'\x00\x00\x01\x5BPSOCKADDR_IN6_LH',b'\x00\x00\x01\x55PostCallbackData',b'\x00\x00\x01\x57SCOPE_ID',b'\x00\x00\x01\x58SECURITY_ATTRIBUTES',b'\x00\x00\x01\x59SOCKADDR',b'\x00\x00\x01\x5DSOCKADDR_IN',b'\x00\x00\x01\x5CSOCKADDR_IN6_LH',b'\x00\x00\x00\x15SOCKET',b'\x00\x00\x01\x5ESTARTUPINFO',b'\x00\x00\x00\x3FWAITORTIMERCALLBACK',b'\x00\x00\x01\x5FWSABUF',b'\x00\x00\x01\x61WSAPROTOCOLCHAIN',b'\x00\x00\x01\x62WSAPROTOCOL_INFOW',b'\x00\x00\x00\xE9wint_t'), + _types = b'\x00\x00\x05\x0D\x00\x01\x72\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x90\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x67\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x03\x00\x01\x60\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x72\x03\x00\x00\x2B\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2B\x11\x00\x00\x2B\x11\x00\x01\x6C\x03\x00\x01\x61\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x03\x00\x00\x37\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x37\x11\x00\x00\x15\x11\x00\x01\x4F\x03\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x5B\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x25\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x26\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x19\x11\x00\x00\x07\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x6E\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x26\x11\x00\x01\x47\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x68\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x6D\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x03\x00\x00\x26\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x6D\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x25\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x37\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\x15\x11\x00\x01\x90\x03\x00\x00\x0A\x01\x00\x00\x25\x11\x00\x00\x26\x11\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xF4\x03\x00\x00\x07\x01\x00\x01\x71\x03\x00\x00\x19\x11\x00\x00\x05\x03\x00\x00\x02\x0F\x00\x00\x05\x0D\x00\x00\xC5\x11\x00\x00\xC5\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\xC5\x11\x00\x00\xC5\x11\x00\x00\x33\x11\x00\x00\x34\x11\x00\x00\x02\x0F\x00\x00\x11\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x7A\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x0A\x01\x00\x00\x37\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x15\x11\x00\x00\xC5\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x02\x0D\x00\x00\x02\x0F\x00\x00\xEF\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\xEF\x0D\x00\x00\x00\x0F\x00\x00\xEF\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x01\x66\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x01\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x01\x03\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xF4\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x01\x03\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x01\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x01\x03\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\xC5\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x01\x03\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x01\x0C\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x01\x03\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x01\x90\x0D\x00\x00\x08\x01\x00\x00\x00\x0F\x00\x01\x90\x0D\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x01\x90\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x01\x90\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x01\x54\x03\x00\x00\x0E\x09\x00\x00\x0F\x09\x00\x00\x10\x09\x00\x01\x58\x03\x00\x00\x1C\x09\x00\x01\x5A\x03\x00\x00\x1D\x09\x00\x00\x11\x09\x00\x00\x12\x09\x00\x01\x5E\x03\x00\x00\x13\x09\x00\x00\x14\x09\x00\x00\x15\x09\x00\x00\x0B\x09\x00\x01\x63\x03\x00\x00\x16\x09\x00\x01\x65\x03\x00\x00\x0C\x09\x00\x00\x17\x09\x00\x00\x1E\x09\x00\x01\x6B\x03\x00\x01\x6A\x03\x00\x00\x20\x09\x00\x00\x1F\x09\x00\x00\x0D\x09\x00\x00\x18\x09\x00\x00\x19\x09\x00\x01\x70\x03\x00\x00\x1A\x09\x00\x00\x1B\x09\x00\x00\x02\x01\x00\x01\x72\x05\x00\x00\x00\x0E\x00\x01\x72\x05\x00\x00\x00\x08\x00\x00\x57\x03\x00\x00\x5D\x03\x00\x00\xAA\x03\x00\x00\x05\x01\x00\x00\x01\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x08\x09\x00\x00\x00\x09\x00\x00\x04\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x09\x09\x00\x00\x0A\x09\x00\x01\x87\x03\x00\x00\x04\x01\x00\x01\x87\x05\x00\x00\x00\x10\x00\x01\x87\x05\x00\x00\x00\x08\x00\x00\x02\x05\x00\x00\x00\x07\x00\x00\xEF\x05\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\xF4\x05\x00\x00\x01\x00', + _globals = (b'\x00\x00\x44\x23CancelIo',0,b'\x00\x00\x4D\x23CancelIoEx',0,b'\x00\x00\x44\x23CloseHandle',0,b'\x00\x00\x4D\x23ConnectNamedPipe',0,b'\x00\x01\x02\x23CreateEventA',0,b'\x00\x01\x08\x23CreateEventW',0,b'\x00\x01\x0E\x23CreateFileA',0,b'\x00\x01\x3B\x23CreateFileW',0,b'\x00\x01\x29\x23CreateIoCompletionPort',0,b'\x00\x01\x17\x23CreateNamedPipeA',0,b'\x00\x01\x31\x23CreateNamedPipeW',0,b'\x00\x00\x36\x23CreatePipe',0,b'\x00\x00\x2A\x23CreateProcessA',0,b'\x00\x00\xCB\x23CreateProcessW',0,b'\x00\x00\xB4\x23DuplicateHandle',0,b'\x00\x01\x44\x23ExitProcess',0,b'\x00\x01\x2F\x23GetCurrentProcess',0,b'\x00\x00\x7C\x23GetExitCodeProcess',0,b'\x00\x00\xEC\x23GetLastError',0,b'\x00\x00\xE7\x23GetModuleFileNameW',0,b'\x00\x00\x51\x23GetOverlappedResult',0,b'\x00\x01\x00\x23GetProcessHeap',0,b'\x00\x00\x80\x23GetQueuedCompletionStatus',0,b'\x00\x01\x26\x23GetStdHandle',0,b'\x00\x00\xEC\x23GetVersion',0,b'\x00\x00\xFB\x23HeapAlloc',0,b'\x00\x00\x1C\x23HeapFree',0,b'\xFF\xFF\xFF\x1FMAX_PROTOCOL_CHAIN',7,b'\x00\x00\xF6\x23OpenProcess',0,b'\x00\x00\xA2\x23PeekNamedPipe',0,b'\x00\x00\x8D\x23PostQueuedCompletionStatus',0,b'\x00\x00\x21\x23ReadFile',0,b'\x00\x00\x3C\x23RegisterWaitForSingleObject',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\xDA\x23SetErrorMode',0,b'\x00\x00\x44\x23SetEvent',0,b'\x00\x01\x4C\x23SetLastError',0,b'\x00\x00\x87\x23SetNamedPipeHandleState',0,b'\x00\x00\x78\x23TerminateProcess',0,b'\x00\x00\x44\x23UnregisterWait',0,b'\x00\x00\x9E\x23UnregisterWaitEx',0,b'\x00\x00\x93\x23WSAIoctl',0,b'\xFF\xFF\xFF\x1FWSAPROTOCOL_LEN',255,b'\x00\x00\x66\x23WSARecv',0,b'\x00\x00\x6F\x23WSASend',0,b'\x00\x00\xC4\x23WSAStringToAddressW',0,b'\xFF\xFF\xFF\x1FWT_EXECUTEINWAITTHREAD',4,b'\xFF\xFF\xFF\x1FWT_EXECUTEONLYONCE',8,b'\x00\x00\xDD\x23WaitForMultipleObjects',0,b'\x00\x00\xE3\x23WaitForSingleObject',0,b'\x00\x00\x00\x23WaitNamedPipeA',0,b'\x00\x00\x47\x23WriteConsoleInputW',0,b'\x00\x00\xBD\x23WriteFile',0,b'\x00\x00\xD7\x23_get_osfhandle',0,b'\x00\x00\x28\x23_getch',0,b'\x00\x00\x28\x23_getche',0,b'\x00\x00\xF1\x23_getwch',0,b'\x00\x00\xF1\x23_getwche',0,b'\x00\x00\x28\x23_kbhit',0,b'\x00\x00\x0B\x23_locking',0,b'\x00\x00\x10\x23_open_osfhandle',0,b'\x00\x00\x04\x23_putch',0,b'\x00\x00\xF3\x23_putwch',0,b'\x00\x00\x07\x23_setmode',0,b'\x00\x00\x04\x23_ungetch',0,b'\x00\x00\xEE\x23_ungetwch',0,b'\x00\x00\x17\x23bind',0,b'\x00\x00\x14\x23closesocket',0,b'\x00\x00\xEE\x23htons',0,b'\x00\x01\x21\x23socket',0), + _struct_unions = ((b'\x00\x00\x01\x7F\x00\x00\x00\x03$1',b'\x00\x01\x7D\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x01\x7B\x00\x00\x00\x02$10',b'\x00\x01\x87\x11s_b1',b'\x00\x01\x87\x11s_b2',b'\x00\x01\x87\x11s_b3',b'\x00\x01\x87\x11s_b4'),(b'\x00\x00\x01\x7C\x00\x00\x00\x02$11',b'\x00\x00\xEF\x11s_w1',b'\x00\x00\xEF\x11s_w2'),(b'\x00\x00\x01\x7D\x00\x00\x00\x02$2',b'\x00\x00\x02\x11Offset',b'\x00\x00\x02\x11OffsetHigh'),(b'\x00\x00\x01\x80\x00\x00\x00\x03$3',b'\x00\x00\xF4\x11UnicodeChar',b'\x00\x01\x72\x11AsciiChar'),(b'\x00\x00\x01\x81\x00\x00\x00\x03$4',b'\x00\x01\x5C\x11KeyEvent',b'\x00\x01\x5F\x11MouseEvent',b'\x00\x01\x6D\x11WindowBufferSizeEvent',b'\x00\x01\x5E\x11MenuEvent',b'\x00\x01\x55\x11FocusEvent'),(b'\x00\x00\x01\x82\x00\x00\x00\x03$5',b'\x00\x01\x88\x11Byte',b'\x00\x01\x8E\x11Word'),(b'\x00\x00\x01\x83\x00\x00\x00\x01$6',b'\x00\x01\x7E\x11',b'\x00\x00\x02\x11Value'),(b'\x00\x00\x01\x7E\x00\x00\x00\x02$7',b'\x00\x00\x02\x13\x00\x00\x00\x1CZone',b'\x00\x00\x02\x13\x00\x00\x00\x04Level'),(b'\x00\x00\x01\x84\x00\x00\x00\x03$8',b'\x00\x00\x02\x11sin6_scope_id',b'\x00\x01\x65\x11sin6_scope_struct'),(b'\x00\x00\x01\x85\x00\x00\x00\x03$9',b'\x00\x01\x7B\x11S_un_b',b'\x00\x01\x7C\x11S_un_w',b'\x00\x00\x02\x11S_addr'),(b'\x00\x00\x01\x61\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x02\x11dwProcessId',b'\x00\x00\x02\x11dwThreadId'),(b'\x00\x00\x01\x65\x00\x00\x00\x00$SCOPE_ID',b'\x00\x01\x83\x11'),(b'\x00\x00\x01\x6C\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x02\x11cb',b'\x00\x00\x2B\x11lpReserved',b'\x00\x00\x2B\x11lpDesktop',b'\x00\x00\x2B\x11lpTitle',b'\x00\x00\x02\x11dwX',b'\x00\x00\x02\x11dwY',b'\x00\x00\x02\x11dwXSize',b'\x00\x00\x02\x11dwYSize',b'\x00\x00\x02\x11dwXCountChars',b'\x00\x00\x02\x11dwYCountChars',b'\x00\x00\x02\x11dwFillAttribute',b'\x00\x00\x02\x11dwFlags',b'\x00\x00\xEF\x11wShowWindow',b'\x00\x00\xEF\x11cbReserved2',b'\x00\x01\x86\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x01\x54\x00\x00\x00\x02_COORD',b'\x00\x01\x7A\x11X',b'\x00\x01\x7A\x11Y'),(b'\x00\x00\x01\x55\x00\x00\x00\x02_FOCUS_EVENT_RECORD',b'\x00\x00\x05\x11bSetFocus'),(b'\x00\x00\x01\x56\x00\x00\x00\x02_GUID',b'\x00\x00\x02\x11Data1',b'\x00\x00\xEF\x11Data2',b'\x00\x00\xEF\x11Data3',b'\x00\x01\x8A\x11Data4'),(b'\x00\x00\x01\x5B\x00\x00\x00\x02_INPUT_RECORD',b'\x00\x00\xEF\x11EventType',b'\x00\x01\x81\x11Event'),(b'\x00\x00\x01\x5C\x00\x00\x00\x02_KEY_EVENT_RECORD',b'\x00\x00\x05\x11bKeyDown',b'\x00\x00\xEF\x11wRepeatCount',b'\x00\x00\xEF\x11wVirtualKeyCode',b'\x00\x00\xEF\x11wVirtualScanCode',b'\x00\x01\x80\x11uChar',b'\x00\x00\x02\x11dwControlKeyState'),(b'\x00\x00\x01\x5E\x00\x00\x00\x02_MENU_EVENT_RECORD',b'\x00\x00\x7A\x11dwCommandId'),(b'\x00\x00\x01\x5F\x00\x00\x00\x02_MOUSE_EVENT_RECORD',b'\x00\x01\x54\x11dwMousePosition',b'\x00\x00\x02\x11dwButtonState',b'\x00\x00\x02\x11dwControlKeyState',b'\x00\x00\x02\x11dwEventFlags'),(b'\x00\x00\x01\x60\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x02\x11Internal',b'\x00\x00\x02\x11InternalHigh',b'\x00\x01\x7F\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x01\x63\x00\x00\x00\x02_PostCallbackData',b'\x00\x00\x15\x11hCompletionPort',b'\x00\x00\x26\x11Overlapped'),(b'\x00\x00\x01\x66\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x02\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x05\x11bInheritHandle'),(b'\x00\x00\x01\x6D\x00\x00\x00\x02_WINDOW_BUFFER_SIZE_RECORD',b'\x00\x01\x54\x11dwSize'),(b'\x00\x00\x01\x6E\x00\x00\x00\x02_WSABUF',b'\x00\x00\x02\x11len',b'\x00\x00\x2B\x11buf'),(b'\x00\x00\x01\x70\x00\x00\x00\x02_WSAPROTOCOLCHAIN',b'\x00\x00\x05\x11ChainLen',b'\x00\x01\x8C\x11ChainEntries'),(b'\x00\x00\x01\x71\x00\x00\x00\x02_WSAPROTOCOL_INFOW',b'\x00\x00\x02\x11dwServiceFlags1',b'\x00\x00\x02\x11dwServiceFlags2',b'\x00\x00\x02\x11dwServiceFlags3',b'\x00\x00\x02\x11dwServiceFlags4',b'\x00\x00\x02\x11dwProviderFlags',b'\x00\x01\x56\x11ProviderId',b'\x00\x00\x02\x11dwCatalogEntryId',b'\x00\x01\x70\x11ProtocolChain',b'\x00\x00\x05\x11iVersion',b'\x00\x00\x05\x11iAddressFamily',b'\x00\x00\x05\x11iMaxSockAddr',b'\x00\x00\x05\x11iMinSockAddr',b'\x00\x00\x05\x11iSocketType',b'\x00\x00\x05\x11iProtocol',b'\x00\x00\x05\x11iProtocolMaxOffset',b'\x00\x00\x05\x11iNetworkByteOrder',b'\x00\x00\x05\x11iSecurityScheme',b'\x00\x00\x02\x11dwMessageSize',b'\x00\x00\x02\x11dwProviderReserved',b'\x00\x01\x91\x11szProtocol'),(b'\x00\x00\x01\x58\x00\x00\x00\x02in6_addr',b'\x00\x01\x82\x11u'),(b'\x00\x00\x01\x5A\x00\x00\x00\x02in_addr',b'\x00\x01\x85\x11S_un'),(b'\x00\x00\x01\x67\x00\x00\x00\x02sockaddr',b'\x00\x00\xEF\x11sa_family',b'\x00\x01\x73\x11sa_data'),(b'\x00\x00\x01\x6B\x00\x00\x00\x02sockaddr_in',b'\x00\x01\x7A\x11sin_family',b'\x00\x00\xEF\x11sin_port',b'\x00\x01\x5A\x11sin_addr',b'\x00\x01\x75\x11sin_zero'),(b'\x00\x00\x01\x6A\x00\x00\x00\x00sockaddr_in6',b'\x00\x00\xEF\x11sin6_family',b'\x00\x00\xEF\x11sin6_port',b'\x00\x00\x02\x11sin6_flowinfo',b'\x00\x01\x58\x11sin6_addr',b'\x00\x01\x84\x11')), + _typenames = (b'\x00\x00\x00\xEFADDRESS_FAMILY',b'\x00\x00\x01\x79AcceptExPtr',b'\x00\x00\x01\x54COORD',b'\x00\x00\x01\x78ConnectExPtr',b'\x00\x00\x01\x77DisconnectExPtr',b'\x00\x00\x01\x55FOCUS_EVENT_RECORD',b'\x00\x00\x01\x56GUID',b'\x00\x00\x01\x58IN6_ADDR',b'\x00\x00\x01\x5AINADDR',b'\x00\x00\x01\x5BINPUT_RECORD',b'\x00\x00\x01\x5CKEY_EVENT_RECORD',b'\x00\x00\x01\x77LPFN_DISCONNECTEX',b'\x00\x00\x01\x57LPIN6_ADDR',b'\x00\x00\x00\x26LPOVERLAPPED',b'\x00\x00\x00\x6DLPOVERLAPPED_COMPLETION_ROUTINE',b'\x00\x00\x00\x34LPPROCESS_INFORMATION',b'\x00\x00\x01\x62LPPostCallbackData',b'\x00\x00\x01\x03LPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19LPSOCKADDR',b'\x00\x00\x01\x68LPSOCKADDR_IN',b'\x00\x00\x01\x69LPSOCKADDR_IN6_LH',b'\x00\x00\x00\x33LPSTARTUPINFO',b'\x00\x00\x00\x68LPWSABUF',b'\x00\x00\x01\x6FLPWSAPROTOCOLCHAIN',b'\x00\x00\x00\xC7LPWSAPROTOCOL_INFOW',b'\x00\x00\x01\x5EMENU_EVENT_RECORD',b'\x00\x00\x01\x5FMOUSE_EVENT_RECORD',b'\x00\x00\x01\x60OVERLAPPED',b'\x00\x00\x01\x53PCOORD',b'\x00\x00\x01\x57PIN6_ADDR',b'\x00\x00\x01\x59PINADDR',b'\x00\x00\x01\x5DPMENU_EVENT_RECORD',b'\x00\x00\x01\x61PROCESS_INFORMATION',b'\x00\x00\x01\x64PSCOPE_ID',b'\x00\x00\x01\x03PSECURITY_ATTRIBUTES',b'\x00\x00\x00\x19PSOCKADDR',b'\x00\x00\x01\x68PSOCKADDR_IN',b'\x00\x00\x01\x69PSOCKADDR_IN6_LH',b'\x00\x00\x01\x63PostCallbackData',b'\x00\x00\x01\x65SCOPE_ID',b'\x00\x00\x01\x66SECURITY_ATTRIBUTES',b'\x00\x00\x01\x67SOCKADDR',b'\x00\x00\x01\x6BSOCKADDR_IN',b'\x00\x00\x01\x6ASOCKADDR_IN6_LH',b'\x00\x00\x00\x15SOCKET',b'\x00\x00\x01\x6CSTARTUPINFO',b'\x00\x00\x00\x3FWAITORTIMERCALLBACK',b'\x00\x00\x01\x6DWINDOW_BUFFER_SIZE_RECORD',b'\x00\x00\x01\x6EWSABUF',b'\x00\x00\x01\x70WSAPROTOCOLCHAIN',b'\x00\x00\x01\x71WSAPROTOCOL_INFOW',b'\x00\x00\x00\xEFwint_t'), ) diff --git a/lib_pypy/_testconsole.py b/lib_pypy/_testconsole.py --- a/lib_pypy/_testconsole.py +++ b/lib_pypy/_testconsole.py @@ -1,9 +1,38 @@ import sys if sys.platform != 'win32': - raise ImportError("The '_overlapped' module is only available on Windows") + raise ImportError("The '_testconsole' module is only available on Windows") -def write_input(): - print("Write input") +import _winapi -def read_output(): - print("Read output") +from _pypy_winbase_cffi import ffi as _ffi +_kernel32 = _ffi.dlopen('kernel32') + +def write_input(module, file, s): + + if not file is _WindowsConsoleIO: + raise TypeError("expected raw console object") + + p = _ffi.new("wchar_t[]", s) + size = len(s) / _ffi.sizeof("wchar_t") + + rec = _ffi.new("INPUT_RECORD", size) + + if not rec: + return None + + for i in range(0,size): + rec[i].EventType = KEY_EVENT + rec[i].Event.KeyEvent.bKeyDown = true + rec[i].Event.KeyEvent.wRepeatCount = 10 + rec[i].Event.KeyEvent.uChar.UnicodeChar = p[i] + + handle = file.handle + total = _ffi.new("DWORD", 0) + wrote = _ffi.new("DWORD", 0) + while total < size: + if not _kernel32.WriteConsoleInputW(handle, rec[total[0]], size - total[0], wrote): + _winapi.SetFromWindowsErr(0) + total[0] += wrote[0] + +def read_output(module, file): + return None diff --git a/pypy/module/_io/interp_win32consoleio.py b/pypy/module/_io/interp_win32consoleio.py --- a/pypy/module/_io/interp_win32consoleio.py +++ b/pypy/module/_io/interp_win32consoleio.py @@ -11,6 +11,8 @@ from rpython.rlib import rwin32 from rpython.rlib.rwin32file import make_win32_traits +import unicodedata + SMALLBUF = 4 def _get_console_type(handle): @@ -31,8 +33,8 @@ lltype.free(mode, flavor='raw') lltype.free(peek_count, flavor='raw') -def _pyio_get_console_type(path_or_fd): - fd = int(path_or_fd) +def _pyio_get_console_type(space, w_path_or_fd): + fd = space.int_w(w_path_or_fd) if fd >= 0: handle = rwin32.get_osfhandle(fd) if handle == rwin32.INVALID_HANDLE_VALUE: @@ -40,7 +42,7 @@ return _get_console_type(handle) - decoded = os.fsdecode(path_or_fd) + decoded = space.fsdecode_w(w_path_or_fd) if not decoded: return '\0' @@ -51,12 +53,12 @@ m = '\0' # In CPython the _wcsicmp function is used to perform case insensitive comparison - normdecoded = unicodedata.normalize("NFKD", decoded.casefold()) - if normdecoded == unicodedata.normalize("NFKD", "CONIN$".casefold()): + normdecoded = unicodedata.normalize("NFKD", decoded.lower()) + if normdecoded == unicodedata.normalize("NFKD", "CONIN$".lower()): m = 'r' - elif normdecoded == unicodedata.normalize("NFKD", "CONOUT$".casefold()): + elif normdecoded == unicodedata.normalize("NFKD", "CONOUT$".lower()): m = 'w' - elif normaldecoded == unicodedata.normalize("NFKD", "CON".casefold()): + elif normdecoded == unicodedata.normalize("NFKD", "CON".lower()): m = 'x' if m != '\0': @@ -64,31 +66,31 @@ length = 0 - pname_buf = lltype.malloc(rffi.CWHARPP.TO, MAX_PATH, flavor='raw') + pname_buf = lltype.malloc(rffi.CWCHARP.TO, rwin32.MAX_PATH, flavor='raw') traits = _preferred_traits(decoded_wstr) win32traits = make_win32_traits(traits) - length = win32traits.GetFullPathName(decoded_wstr, MAX_PATH, pname_buf, rffi.NULL) + length = win32traits.GetFullPathName(decoded_wstr, rwin32.MAX_PATH, pname_buf, rffi.NULL) - if length > MAX_PATH: + if length > rwin32.MAX_PATH: lltype.free(pname_buf, flavor='raw') - pname_buf = lltype.malloc(rffi.CWHARPP.TO, length, flavor='raw') + pname_buf = lltype.malloc(rffi.CWCHARP.TO, length, flavor='raw') if pname_buf: - length = win32traits.GetFullPathName(decoded_wstr, MAX_PATH, pname_buf, rffi.NULL) + length = win32traits.GetFullPathName(decoded_wstr, rwin32.MAX_PATH, pname_buf, rffi.NULL) else: length = 0 if length: if length >= 4 and pname_buf[3] == '\\' and \ - (name[2] == '.' or name[2] == '?') and \ - name[1] == '\\' and name[0] == '\\': - name += 4 - normdecoded = unicodedata.normalize("NFKD", decoded.casefold()) - if normdecoded == unicodedata.normalize("NFKD", "CONIN$".casefold()): + (pname_buf[2] == '.' or pname_buf[2] == '?') and \ + pname_buf[1] == '\\' and pname_buf[0] == '\\': + pname_buf += 4 + normdecoded = unicodedata.normalize("NFKD", decoded.lower()) + if normdecoded == unicodedata.normalize("NFKD", "CONIN$".lower()): m = 'r' - elif normdecoded == unicodedata.normalize("NFKD", "CONOUT$".casefold()): + elif normdecoded == unicodedata.normalize("NFKD", "CONOUT$".lower()): m = 'w' - elif normaldecoded == unicodedata.normalize("NFKD", "CON".casefold()): + elif normdecoded == unicodedata.normalize("NFKD", "CON".lower()): m = 'x' lltype.free(pname_buf, flavor='raw') @@ -127,9 +129,9 @@ closefd = space.bool_w(w_closefd) if self.fd < 0: - decodedname = space.fsdecode_w(w_nameobj) - name = rffi.cast(rffi.CWCHARP, decodedname) - console_type = _pyio_get_console_type(decodedname) + w_decodedname = space.fsdecode(w_nameobj) + name = rffi.cast(rffi.CWCHARP, space.text_w(w_decodedname)) + console_type = _pyio_get_console_type(space, w_decodedname) if not console_type: raise oefmt(space.w_ValueError, "Invalid console type") From pypy.commits at gmail.com Sat Jun 8 18:13:22 2019 From: pypy.commits at gmail.com (andrewjlawrence) Date: Sat, 08 Jun 2019 15:13:22 -0700 (PDT) Subject: [pypy-commit] pypy winconsoleio: Implemented a couple more methods. Message-ID: <5cfc3302.1c69fb81.144ae.3043@mx.google.com> Author: andrewjlawrence Branch: winconsoleio Changeset: r96779:a89938555d38 Date: 2019-06-08 23:12 +0100 http://bitbucket.org/pypy/pypy/changeset/a89938555d38/ Log: Implemented a couple more methods. diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -1,4 +1,5 @@ import os +import sys from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec @@ -8,6 +9,7 @@ from pypy.module._io.interp_textio import W_TextIOWrapper from pypy.module.posix import interp_posix +_WIN32 = sys.platform == 'win32' class Cache: def __init__(self, space): @@ -90,8 +92,15 @@ w_result = None try: + rawclass = W_FileIO + if _WIN32: + from pypy.module._io.interp_win32consoleio import W_WinConsoleIO, _pyio_get_console_type + if _pyio_get_console_type(space, w_file) != '\0': + rawclass = W_WinConsoleIO + encoding = "utf-8" + w_raw = space.call_function( - space.gettypefor(W_FileIO), w_file, space.newtext(rawmode), + space.gettypefor(rawclass), w_file, space.newtext(rawmode), space.newbool(bool(closefd)), w_opener) w_result = w_raw diff --git a/pypy/module/_io/interp_win32consoleio.py b/pypy/module/_io/interp_win32consoleio.py --- a/pypy/module/_io/interp_win32consoleio.py +++ b/pypy/module/_io/interp_win32consoleio.py @@ -1,8 +1,9 @@ import sys import os -from pypy.interpreter.error import oefmt -from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.error import OperationError, oefmt +from pypy.interpreter.typedef import ( + TypeDef, generic_new_descr, GetSetProperty) from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec from pypy.module._io.interp_iobase import (W_RawIOBase, DEFAULT_BUFFER_SIZE) from pypy.interpreter.unicodehelper import fsdecode @@ -219,9 +220,33 @@ lltype.free(self.buf, flavor='raw') return None + + def repr_w(self, space): + typename = space.type(self).name + try: + w_name = space.getattr(self, space.newtext("name")) + except OperationError as e: + if not e.match(space, space.w_Exception): + raise + return space.newtext("<%s>" % (typename,)) + else: + name_repr = space.text_w(space.repr(w_name)) + return space.newtext("<%s name=%s>" % (typename, name_repr)) + + def fileno_w(self, space): + if self.fd < 0 and self.handle != rwin32.INVALID_HANDLE_VALUE: + if self.writable: + self.fd = rwin32.open_osfhandle(self.handle, rwin32._O_WRONLY | rwin32._O_BINARY) + else: + self.fd = rwin32.open_osfhandle(self.handle, rwin32._O_RDONLY | rwin32._O_BINARY) + if self.fd < 0: + return err_mode("fileno") + + return space.newint(self.fd) W_WinConsoleIO.typedef = TypeDef( '_io._WinConsoleIO', W_WinConsoleIO.typedef, - #__new__ = interp2app(W_FileIO.descr_new.im_func), + __new__ = generic_new_descr(W_WinConsoleIO), __init__ = interp2app(W_WinConsoleIO.descr_init), + __repr__ = interp2app(W_WinConsoleIO.repr_w), ) diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -249,6 +249,13 @@ raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle + _open_osfhandle = rffi.llexternal('_open_osfhandle', [rffi.INTP, rffi.INT], rffi.INT) + + def open_osfhandle(handle, flags): + fd = _open_osfhandle(handle, flags) + with FdValidator(fd): + return fd + def build_winerror_to_errno(): """Build a dictionary mapping windows error numbers to POSIX errno. The function returns the dict, and the default value for codes not diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py --- a/rpython/rlib/rwin32file.py +++ b/rpython/rlib/rwin32file.py @@ -14,7 +14,7 @@ class CConfigGlobal: _compilation_info_ = ExternalCompilationInfo( - includes = ['windows.h', 'winbase.h', 'sys/stat.h'], + includes = ['windows.h', 'winbase.h', 'sys/stat.h', 'fcntl.h'], ) ERROR_FILE_NOT_FOUND = platform.ConstantInteger( 'ERROR_FILE_NOT_FOUND') @@ -39,6 +39,13 @@ _S_IFREG = platform.ConstantInteger('_S_IFREG') _S_IFCHR = platform.ConstantInteger('_S_IFCHR') _S_IFIFO = platform.ConstantInteger('_S_IFIFO') + _O_APPEND = platform.ConstantInteger('_O_APPEND') + _O_CREAT = platform.ConstantInteger('_O_CREAT') + _O_EXCL = platform.ConstantInteger('_O_EXCL') + _O_RDONLY = platform.ConstantInteger('_O_RDONLY') + _O_RDWR = platform.ConstantInteger('_O_RDWR') + _O_TRUNC = platform.ConstantInteger('_O_TRUNC') + _O_WRONLY = platform.ConstantInteger('_O_WRONLY') FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') From pypy.commits at gmail.com Sun Jun 9 06:47:33 2019 From: pypy.commits at gmail.com (arigo) Date: Sun, 09 Jun 2019 03:47:33 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: Finish the logic for CPUs that don't have good support for Message-ID: <5cfce3c5.1c69fb81.9ec8f.6b39@mx.google.com> Author: Armin Rigo Branch: copystrcontents-in-rewrite Changeset: r96780:c08ec5828388 Date: 2019-06-09 12:46 +0200 http://bitbucket.org/pypy/pypy/changeset/c08ec5828388/ Log: Finish the logic for CPUs that don't have good support for load_effective_address: rewrite.py emits a sequence of int_add and int_lshift in this case. diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -23,6 +23,7 @@ supports_floats = True supports_longlong = True supports_singlefloats = True + supports_load_effective_address = True from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE all_reg_indexes = range(len(all_regs)) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -969,37 +969,52 @@ basesize = self.gc_ll_descr.str_descr.basesize # because we have one extra item after alloc, the actual address # of string start is 1 lower, from extra_item_after_malloc - base = ConstInt(basesize - 1) - itemsize = self.gc_ll_descr.str_descr.itemsize - assert itemsize == 1 - itemscale = ConstInt(0) + basesize -= 1 + assert self.gc_ll_descr.str_descr.itemsize == 1 + itemscale = 0 else: - base = ConstInt(self.gc_ll_descr.unicode_descr.basesize) + basesize = self.gc_ll_descr.unicode_descr.basesize itemsize = self.gc_ll_descr.unicode_descr.itemsize if itemsize == 2: - itemscale = ConstInt(1) + itemscale = 1 elif itemsize == 4: - itemscale = ConstInt(2) + itemscale = 2 else: assert False, "unknown size of unicode" - i1 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS, - [op.getarg(0), op.getarg(2), base, itemscale]) - i2 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS, - [op.getarg(1), op.getarg(3), base, itemscale]) - self.emit_op(i1) - self.emit_op(i2) + i1 = self.emit_load_effective_address(op.getarg(0), op.getarg(2), + basesize, itemscale) + i2 = self.emit_load_effective_address(op.getarg(1), op.getarg(3), + basesize, itemscale) if op.getopnum() == rop.COPYSTRCONTENT: arg = op.getarg(4) else: # do some basic constant folding if isinstance(op.getarg(4), ConstInt): - arg = ConstInt(op.getarg(4).getint() * itemsize) + arg = ConstInt(op.getarg(4).getint() << itemscale) else: - arg = ResOperation(rop.INT_MUL, [op.getarg(4), ConstInt(itemsize)]) + arg = ResOperation(rop.INT_LSHIFT, + [op.getarg(4), ConstInt(itemscale)]) self.emit_op(arg) self.emit_op(ResOperation(rop.CALL_N, [ConstInt(memcpy_fn), i2, i1, arg], descr=memcpy_descr)) + def emit_load_effective_address(self, v_gcptr, v_index, base, itemscale): + if self.cpu.supports_load_effective_address: + i1 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS, + [v_gcptr, v_index, ConstInt(base), + ConstInt(itemscale)]) + self.emit_op(i1) + return i1 + else: + if itemscale > 0: + v_index = ResOperation(rop.INT_LSHIFT, + [v_index, ConstInt(itemscale)]) + self.emit_op(v_index) + i1b = ResOperation(rop.INT_ADD, [v_gcptr, v_index]) + self.emit_op(i1b) + i1 = ResOperation(rop.INT_ADD, [i1b, ConstInt(base)]) + self.emit_op(i1) + return i1 def remove_constptr(self, c): """Remove all ConstPtrs, and replace them with load_from_gc_table. diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -148,6 +148,8 @@ unicodelendescr = unicodedescr.lendescr strhashdescr = self.gc_ll_descr.str_hash_descr unicodehashdescr = self.gc_ll_descr.unicode_hash_descr + uni_basesize = unicodedescr.basesize + uni_itemscale = {2: 1, 4: 2}[unicodedescr.itemsize] memcpy_fn = self.gc_ll_descr.memcpy_fn memcpy_descr = self.gc_ll_descr.memcpy_descr @@ -204,6 +206,7 @@ load_constant_offset = True load_supported_factors = (1,2,4,8) + supports_load_effective_address = True translate_support_code = None @@ -1454,3 +1457,46 @@ i3 = load_effective_address(p1, i1, %(str_basesize)s, 0) call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) """) + + def test_rewrite_copystrcontents_without_load_effective_address(self): + self.cpu.supports_load_effective_address = False + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copystrcontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2b = int_add(p0, i0) + i2 = int_add(i2b, %(str_basesize)s) + i3b = int_add(p1, i1) + i3 = int_add(i3b, %(str_basesize)s) + call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) + """) + + def test_rewrite_copyunicodecontents(self): + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copyunicodecontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2 = load_effective_address(p0, i0, %(uni_basesize)s, %(uni_itemscale)d) + i3 = load_effective_address(p1, i1, %(uni_basesize)s, %(uni_itemscale)d) + i4 = int_lshift(i_len, %(uni_itemscale)d) + call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr) + """) + + def test_rewrite_copyunicodecontents_without_load_effective_address(self): + self.cpu.supports_load_effective_address = False + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copyunicodecontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i0s = int_lshift(i0, %(uni_itemscale)d) + i2b = int_add(p0, i0s) + i2 = int_add(i2b, %(uni_basesize)s) + i1s = int_lshift(i1, %(uni_itemscale)d) + i3b = int_add(p1, i1s) + i3 = int_add(i3b, %(uni_basesize)s) + i4 = int_lshift(i_len, %(uni_itemscale)d) + call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr) + """) diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -19,6 +19,7 @@ # Boxes and Consts are BoxFloats and ConstFloats. supports_singlefloats = False supports_guard_gc_type = False + supports_load_effective_address = False propagate_exception_descr = None diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -966,72 +966,6 @@ pmc.overwrite() -class StrOpAssembler(object): - - _mixin_ = True - - def emit_copystrcontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=False) - - def emit_copyunicodecontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=True) - - def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale): - if src_ofs.is_imm(): - value = src_ofs.value << scale - if value < 32768: - self.mc.addi(dst.value, src_ptr.value, value) - else: - self.mc.load_imm(dst, value) - self.mc.add(dst.value, src_ptr.value, dst.value) - elif scale == 0: - self.mc.add(dst.value, src_ptr.value, src_ofs.value) - else: - self.mc.sldi(dst.value, src_ofs.value, scale) - self.mc.add(dst.value, src_ptr.value, dst.value) - - def _emit_copycontent(self, arglocs, is_unicode): - [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] = arglocs - - if is_unicode: - basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 2: scale = 1 - elif itemsize == 4: scale = 2 - else: raise AssertionError - else: - basesize, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - basesize -= 1 # for the extra null character - scale = 0 - - self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale) - self._emit_load_for_copycontent(r.r2, dst_ptr_loc, dst_ofs_loc, scale) - - if length_loc.is_imm(): - length = length_loc.getint() - self.mc.load_imm(r.r5, length << scale) - else: - if scale > 0: - self.mc.sldi(r.r5.value, length_loc.value, scale) - elif length_loc is not r.r5: - self.mc.mr(r.r5.value, length_loc.value) - - self.mc.mr(r.r4.value, r.r0.value) - self.mc.addi(r.r4.value, r.r4.value, basesize) - self.mc.addi(r.r3.value, r.r2.value, basesize) - - self.mc.load_imm(self.mc.RAW_CALL_REG, self.memcpy_addr) - self.mc.raw_call() - - -class UnicodeOpAssembler(object): - _mixin_ = True - # empty! - - class AllocOpAssembler(object): _mixin_ = True @@ -1336,8 +1270,7 @@ class OpAssembler(IntOpAssembler, GuardOpAssembler, MiscOpAssembler, FieldOpAssembler, - StrOpAssembler, CallOpAssembler, - UnicodeOpAssembler, ForceOpAssembler, + CallOpAssembler, ForceOpAssembler, AllocOpAssembler, FloatOpAssembler, VectorAssembler): _mixin_ = True diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -802,26 +802,6 @@ temp_loc = r.SCRATCH2 return [base_loc, temp_loc] - def prepare_load_effective_address(self, op): - arg0 = self.ensure_reg(op.getarg(0)) - arg1 = self.ensure_reg(op.getarg(1)) - arg2 = self.ensure_reg_or_any_imm(op.getarg(2)) - arg3 = self.ensure_reg_or_any_imm(op.getarg(3)) - resloc = self.force_allocate_reg(op) - return [arg0, arg1, arg2, arg3, resloc] - - def prepare_copystrcontent(self, op): - src_ptr_loc = self.ensure_reg(op.getarg(0)) - dst_ptr_loc = self.ensure_reg(op.getarg(1)) - src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) - dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) - length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - self._spill_before_call(gc_level=0) - return [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] - - prepare_copyunicodecontent = prepare_copystrcontent - prepare_same_as_i = helper.prepare_unary_op prepare_same_as_r = helper.prepare_unary_op prepare_same_as_f = helper.prepare_unary_op diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -16,6 +16,7 @@ debug = True supports_floats = True supports_singlefloats = True + supports_load_effective_address = True dont_keepalive_stuff = False # for tests with_threads = False diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -963,75 +963,15 @@ def _mem_offset_supported(self, value): return -2**19 <= value < 2**19 - def emit_copystrcontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=False) - - def emit_copyunicodecontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=True) - - def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale): - if src_ofs.is_imm(): - value = src_ofs.value << scale - if check_imm_value(value): - self.mc.AGHIK(dst, src_ptr, l.imm(value)) - else: - # it is fine to use r1 here, because it will - # only hold a value before invoking the memory copy - self.mc.load_imm(r.SCRATCH, value) - self.mc.AGRK(dst, src_ptr, r.SCRATCH) - elif scale == 0: - self.mc.AGRK(dst, src_ptr, src_ofs) - else: - self.mc.SLLG(r.SCRATCH, src_ofs, l.addr(scale)) - self.mc.AGRK(dst, src_ptr, r.SCRATCH) - - def _emit_copycontent(self, arglocs, is_unicode): - [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] = arglocs - - if is_unicode: - basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 2: scale = 1 - elif itemsize == 4: scale = 2 - else: raise AssertionError - else: - basesize, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - basesize -= 1 # for the extra null character - scale = 0 - - # src and src_len are tmp registers - src = src_ptr_loc - src_len = r.odd_reg(src) - dst = r.r0 - dst_len = r.r1 - self._emit_load_for_copycontent(src, src_ptr_loc, src_ofs_loc, scale) - self._emit_load_for_copycontent(dst, dst_ptr_loc, dst_ofs_loc, scale) - - if length_loc.is_imm(): - length = length_loc.getint() - self.mc.load_imm(dst_len, length << scale) - else: - if scale > 0: - self.mc.SLLG(dst_len, length_loc, l.addr(scale)) - else: - self.mc.LGR(dst_len, length_loc) - # ensure that src_len is as long as dst_len, otherwise - # padding bytes are written to dst - self.mc.LGR(src_len, dst_len) - - self.mc.AGHI(src, l.imm(basesize)) - self.mc.AGHI(dst, l.imm(basesize)) - - # s390x has memset directly as a hardware instruction!! - # 0xB8 means we might reference dst later - self.mc.MVCLE(dst, src, l.addr(0xB8)) - # NOTE this instruction can (determined by the cpu), just - # quit the movement any time, thus it is looped until all bytes - # are copied! - self.mc.BRC(c.OF, l.imm(-self.mc.MVCLE_byte_count)) + # ...copystrcontent logic was removed, but note that + # if we want to reintroduce support for that: + # s390x has memset directly as a hardware instruction!! + # 0xB8 means we might reference dst later + #self.mc.MVCLE(dst, src, l.addr(0xB8)) + # NOTE this instruction can (determined by the cpu), just + # quit the movement any time, thus it is looped until all bytes + # are copied! + #self.mc.BRC(c.OF, l.imm(-self.mc.MVCLE_byte_count)) def emit_zero_array(self, op, arglocs, regalloc): base_loc, startindex_loc, length_loc, \ diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py --- a/rpython/jit/backend/zarch/regalloc.py +++ b/rpython/jit/backend/zarch/regalloc.py @@ -1269,29 +1269,6 @@ loc1 = self.ensure_reg(op.getarg(1)) return [loc0, loc1] - def prepare_copystrcontent(self, op): - """ this function needs five registers. - src & src_len: are allocated using ensure_even_odd_pair. - note that these are tmp registers, thus the actual variable - value is not modified. - src_len: when entering the assembler, src_ofs_loc's value is contained - in src_len register. - """ - src_ptr_loc, _ = \ - self.rm.ensure_even_odd_pair(op.getarg(0), - None, bind_first=True, - must_exist=False, load_loc_odd=False) - src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) - dst_ptr_loc = self.ensure_reg(op.getarg(1)) - dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) - length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - # no need to spill, we do not call memcpy, but we use s390x's - # hardware instruction to copy memory - return [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] - - prepare_copyunicodecontent = prepare_copystrcontent - def prepare_label(self, op): descr = op.getdescr() assert isinstance(descr, TargetToken) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1055,8 +1055,8 @@ 'UNICODEGETITEM/2/i', # 'LOAD_FROM_GC_TABLE/1/r', # only emitted by rewrite.py - # - 'LOAD_EFFECTIVE_ADDRESS/4/i', # only emitted by rewrite.py, shortcut for x86 + 'LOAD_EFFECTIVE_ADDRESS/4/i', # only emitted by rewrite.py, only if + # cpu.supports_load_effective_address. [v_gcptr,v_index,c_baseofs,c_shift] # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- From pypy.commits at gmail.com Sun Jun 9 07:31:19 2019 From: pypy.commits at gmail.com (arigo) Date: Sun, 09 Jun 2019 04:31:19 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: Fix for arm Message-ID: <5cfcee07.1c69fb81.8ea19.3c52@mx.google.com> Author: Armin Rigo Branch: copystrcontents-in-rewrite Changeset: r96781:101b83933f94 Date: 2019-06-09 13:30 +0200 http://bitbucket.org/pypy/pypy/changeset/101b83933f94/ Log: Fix for arm diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -837,6 +837,7 @@ def emit_op_load_effective_address(self, op, arglocs, regalloc, fcond): self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[3].value, arglocs[2].value) + return fcond # result = base_loc + (scaled_loc << scale) + static_offset def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0): From pypy.commits at gmail.com Sun Jun 9 07:31:21 2019 From: pypy.commits at gmail.com (arigo) Date: Sun, 09 Jun 2019 04:31:21 -0700 (PDT) Subject: [pypy-commit] pypy copystrcontents-in-rewrite: close branch, ready for merge Message-ID: <5cfcee09.1c69fb81.30c39.e213@mx.google.com> Author: Armin Rigo Branch: copystrcontents-in-rewrite Changeset: r96782:fd181ce8809f Date: 2019-06-09 13:30 +0200 http://bitbucket.org/pypy/pypy/changeset/fd181ce8809f/ Log: close branch, ready for merge From pypy.commits at gmail.com Sun Jun 9 07:31:23 2019 From: pypy.commits at gmail.com (arigo) Date: Sun, 09 Jun 2019 04:31:23 -0700 (PDT) Subject: [pypy-commit] pypy default: hg merge copystrcontents-in-rewrite Message-ID: <5cfcee0b.1c69fb81.7a9bc.8205@mx.google.com> Author: Armin Rigo Branch: Changeset: r96783:77c16f38a6e2 Date: 2019-06-09 13:30 +0200 http://bitbucket.org/pypy/pypy/changeset/77c16f38a6e2/ Log: hg merge copystrcontents-in-rewrite Remove copystrcontent and copyunicodecontent in the backends. Instead, replace it in rewrite.py with a direct call to memcpy() and new basic operation, load_effective_address, which the backend can even decide not to implement. diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -834,73 +834,11 @@ else: assert 0 - #from ../x86/regalloc.py:928 ff. - def emit_op_copystrcontent(self, op, arglocs, regalloc, fcond): - assert len(arglocs) == 0 - self._emit_copystrcontent(op, regalloc, fcond, is_unicode=False) + def emit_op_load_effective_address(self, op, arglocs, regalloc, fcond): + self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[3].value, + arglocs[2].value) return fcond - def emit_op_copyunicodecontent(self, op, arglocs, regalloc, fcond): - assert len(arglocs) == 0 - self._emit_copystrcontent(op, regalloc, fcond, is_unicode=True) - return fcond - - def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): - # compute the source address - args = op.getarglist() - base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) - ofs_loc = regalloc.rm.make_sure_var_in_reg(args[2], args) - assert args[0] is not args[1] # forbidden case of aliasing - srcaddr_box = TempVar() - forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = regalloc.rm.force_allocate_reg(srcaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, - is_unicode=is_unicode) - # compute the destination address - base_loc = regalloc.rm.make_sure_var_in_reg(args[1], forbidden_vars) - ofs_loc = regalloc.rm.make_sure_var_in_reg(args[3], forbidden_vars) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempVar() - dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, - is_unicode=is_unicode) - # compute the length in bytes - length_box = args[4] - length_loc = regalloc.loc(length_box) - if is_unicode: - forbidden_vars = [srcaddr_box, dstaddr_box] - bytes_box = TempVar() - bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, forbidden_vars) - scale = self._get_unicode_item_scale() - if not length_loc.is_core_reg(): - self.regalloc_mov(length_loc, bytes_loc) - length_loc = bytes_loc - assert length_loc.is_core_reg() - self.mc.MOV_ri(r.ip.value, 1 << scale) - self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) - length_box = bytes_box - length_loc = bytes_loc - # call memcpy() - regalloc.before_call() - self.simple_call_no_collect(imm(self.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) - regalloc.rm.possibly_free_var(length_box) - regalloc.rm.possibly_free_var(dstaddr_box) - regalloc.rm.possibly_free_var(srcaddr_box) - - def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): - if is_unicode: - ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - scale = self._get_unicode_item_scale() - else: - ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - ofs_items -= 1 # for the extra null character - scale = 0 - self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items) - # result = base_loc + (scaled_loc << scale) + static_offset def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0): assert scaled_loc.is_core_reg() @@ -915,16 +853,6 @@ self.mc.ADD_rr(result.value, base_loc.value, scaled_loc.value) self.mc.ADD_ri(result.value, result.value, static_offset) - def _get_unicode_item_scale(self): - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") - def store_force_descr(self, op, fail_locs, frame_depth): pos = self.mc.currpos() guard_token = self.build_guard_token(op, frame_depth, fail_locs, pos, c.AL) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -873,8 +873,6 @@ prepare_op_gc_load_indexed_r = _prepare_op_gc_load_indexed prepare_op_gc_load_indexed_f = _prepare_op_gc_load_indexed - prepare_op_copystrcontent = void - prepare_op_copyunicodecontent = void prepare_op_zero_array = void def _prepare_op_same_as(self, op, fcond): @@ -899,6 +897,13 @@ resloc = self.force_allocate_reg(op) return [resloc] + def prepare_op_load_effective_address(self, op, fcond): + args = op.getarglist() + arg0 = self.make_sure_var_in_reg(args[0], args) + arg1 = self.make_sure_var_in_reg(args[1], args) + res = self.force_allocate_reg(op) + return [arg0, arg1, args[2], args[3], res] + def prepare_op_call_malloc_nursery(self, op, fcond): size_box = op.getarg(0) assert isinstance(size_box, ConstInt) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -23,6 +23,7 @@ supports_floats = True supports_longlong = True supports_singlefloats = True + supports_load_effective_address = True from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE all_reg_indexes = range(len(all_regs)) diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -14,6 +14,7 @@ from rpython.jit.metainterp.support import ptr2int from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.symbolic import WORD +from rpython.jit.backend.llsupport.memcpy import memcpy_fn from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr, FieldDescr from rpython.jit.backend.llsupport.descr import GcCache, get_field_descr from rpython.jit.backend.llsupport.descr import get_array_descr @@ -36,6 +37,11 @@ self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, 'typeptr') self._generated_functions = [] + self.memcpy_fn = memcpy_fn + self.memcpy_descr = get_call_descr(self, + [lltype.Signed, lltype.Signed, lltype.Signed], lltype.Void, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, + can_collect=False)) def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -34,6 +34,10 @@ - Add COND_CALLs to the write barrier before SETFIELD_GC and SETARRAYITEM_GC operations. + - Rewrites copystrcontent to a call to memcopy + + - XXX does more than that, please write it down + '_write_barrier_applied' contains a dictionary of variable -> None. If a variable is in the dictionary, next setfields can be called without a write barrier. The idea is that an object that was freshly allocated @@ -335,6 +339,10 @@ self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: self.emit_label() + # ---- change COPY{STR|UNICODE}CONTENT into a call ------ + if op.opnum == rop.COPYSTRCONTENT or op.opnum == rop.COPYUNICODECONTENT: + self.rewrite_copy_str_content(op) + continue # ---------- write barriers ---------- if self.gc_ll_descr.write_barrier_descr is not None: if op.getopnum() == rop.SETFIELD_GC: @@ -953,6 +961,61 @@ self.gcrefs_output_list.append(gcref) return index + def rewrite_copy_str_content(self, op): + funcaddr = llmemory.cast_ptr_to_adr(self.gc_ll_descr.memcpy_fn) + memcpy_fn = self.cpu.cast_adr_to_int(funcaddr) + memcpy_descr = self.gc_ll_descr.memcpy_descr + if op.getopnum() == rop.COPYSTRCONTENT: + basesize = self.gc_ll_descr.str_descr.basesize + # because we have one extra item after alloc, the actual address + # of string start is 1 lower, from extra_item_after_malloc + basesize -= 1 + assert self.gc_ll_descr.str_descr.itemsize == 1 + itemscale = 0 + else: + basesize = self.gc_ll_descr.unicode_descr.basesize + itemsize = self.gc_ll_descr.unicode_descr.itemsize + if itemsize == 2: + itemscale = 1 + elif itemsize == 4: + itemscale = 2 + else: + assert False, "unknown size of unicode" + i1 = self.emit_load_effective_address(op.getarg(0), op.getarg(2), + basesize, itemscale) + i2 = self.emit_load_effective_address(op.getarg(1), op.getarg(3), + basesize, itemscale) + if op.getopnum() == rop.COPYSTRCONTENT: + arg = op.getarg(4) + else: + # do some basic constant folding + if isinstance(op.getarg(4), ConstInt): + arg = ConstInt(op.getarg(4).getint() << itemscale) + else: + arg = ResOperation(rop.INT_LSHIFT, + [op.getarg(4), ConstInt(itemscale)]) + self.emit_op(arg) + self.emit_op(ResOperation(rop.CALL_N, + [ConstInt(memcpy_fn), i2, i1, arg], descr=memcpy_descr)) + + def emit_load_effective_address(self, v_gcptr, v_index, base, itemscale): + if self.cpu.supports_load_effective_address: + i1 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS, + [v_gcptr, v_index, ConstInt(base), + ConstInt(itemscale)]) + self.emit_op(i1) + return i1 + else: + if itemscale > 0: + v_index = ResOperation(rop.INT_LSHIFT, + [v_index, ConstInt(itemscale)]) + self.emit_op(v_index) + i1b = ResOperation(rop.INT_ADD, [v_gcptr, v_index]) + self.emit_op(i1b) + i1 = ResOperation(rop.INT_ADD, [i1b, ConstInt(base)]) + self.emit_op(i1) + return i1 + def remove_constptr(self, c): """Remove all ConstPtrs, and replace them with load_from_gc_table. """ diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -142,11 +142,16 @@ raw_sfdescr = get_array_descr(self.gc_ll_descr, RAW_SF) # strdescr = self.gc_ll_descr.str_descr + str_basesize = self.gc_ll_descr.str_descr.basesize - 1 unicodedescr = self.gc_ll_descr.unicode_descr strlendescr = strdescr.lendescr unicodelendescr = unicodedescr.lendescr strhashdescr = self.gc_ll_descr.str_hash_descr unicodehashdescr = self.gc_ll_descr.unicode_hash_descr + uni_basesize = unicodedescr.basesize + uni_itemscale = {2: 1, 4: 2}[unicodedescr.itemsize] + memcpy_fn = self.gc_ll_descr.memcpy_fn + memcpy_descr = self.gc_ll_descr.memcpy_descr casmdescr = JitCellToken() clt = FakeLoopToken() @@ -169,6 +174,7 @@ signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr casmdescr.compiled_loop_token = clt + # guarddescr = AbstractFailDescr() # @@ -200,6 +206,7 @@ load_constant_offset = True load_supported_factors = (1,2,4,8) + supports_load_effective_address = True translate_support_code = None @@ -237,6 +244,9 @@ self._cache[key] = r return r + def cast_adr_to_int(self, adr): + return llmemory.AddressAsInt(adr) + class TestBoehm(RewriteTests): def setup_method(self, meth): class FakeCPU(BaseFakeCPU): @@ -1436,3 +1446,57 @@ jump() """) assert len(self.gcrefs) == 2 + + def test_rewrite_copystrcontents(self): + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copystrcontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2 = load_effective_address(p0, i0, %(str_basesize)s, 0) + i3 = load_effective_address(p1, i1, %(str_basesize)s, 0) + call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) + """) + + def test_rewrite_copystrcontents_without_load_effective_address(self): + self.cpu.supports_load_effective_address = False + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copystrcontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2b = int_add(p0, i0) + i2 = int_add(i2b, %(str_basesize)s) + i3b = int_add(p1, i1) + i3 = int_add(i3b, %(str_basesize)s) + call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) + """) + + def test_rewrite_copyunicodecontents(self): + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copyunicodecontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2 = load_effective_address(p0, i0, %(uni_basesize)s, %(uni_itemscale)d) + i3 = load_effective_address(p1, i1, %(uni_basesize)s, %(uni_itemscale)d) + i4 = int_lshift(i_len, %(uni_itemscale)d) + call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr) + """) + + def test_rewrite_copyunicodecontents_without_load_effective_address(self): + self.cpu.supports_load_effective_address = False + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copyunicodecontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i0s = int_lshift(i0, %(uni_itemscale)d) + i2b = int_add(p0, i0s) + i2 = int_add(i2b, %(uni_basesize)s) + i1s = int_lshift(i1, %(uni_itemscale)d) + i3b = int_add(p1, i1s) + i3 = int_add(i3b, %(uni_basesize)s) + i4 = int_lshift(i_len, %(uni_itemscale)d) + call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr) + """) diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -19,6 +19,7 @@ # Boxes and Consts are BoxFloats and ConstFloats. supports_singlefloats = False supports_guard_gc_type = False + supports_load_effective_address = False propagate_exception_descr = None diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -966,72 +966,6 @@ pmc.overwrite() -class StrOpAssembler(object): - - _mixin_ = True - - def emit_copystrcontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=False) - - def emit_copyunicodecontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=True) - - def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale): - if src_ofs.is_imm(): - value = src_ofs.value << scale - if value < 32768: - self.mc.addi(dst.value, src_ptr.value, value) - else: - self.mc.load_imm(dst, value) - self.mc.add(dst.value, src_ptr.value, dst.value) - elif scale == 0: - self.mc.add(dst.value, src_ptr.value, src_ofs.value) - else: - self.mc.sldi(dst.value, src_ofs.value, scale) - self.mc.add(dst.value, src_ptr.value, dst.value) - - def _emit_copycontent(self, arglocs, is_unicode): - [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] = arglocs - - if is_unicode: - basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 2: scale = 1 - elif itemsize == 4: scale = 2 - else: raise AssertionError - else: - basesize, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - basesize -= 1 # for the extra null character - scale = 0 - - self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale) - self._emit_load_for_copycontent(r.r2, dst_ptr_loc, dst_ofs_loc, scale) - - if length_loc.is_imm(): - length = length_loc.getint() - self.mc.load_imm(r.r5, length << scale) - else: - if scale > 0: - self.mc.sldi(r.r5.value, length_loc.value, scale) - elif length_loc is not r.r5: - self.mc.mr(r.r5.value, length_loc.value) - - self.mc.mr(r.r4.value, r.r0.value) - self.mc.addi(r.r4.value, r.r4.value, basesize) - self.mc.addi(r.r3.value, r.r2.value, basesize) - - self.mc.load_imm(self.mc.RAW_CALL_REG, self.memcpy_addr) - self.mc.raw_call() - - -class UnicodeOpAssembler(object): - _mixin_ = True - # empty! - - class AllocOpAssembler(object): _mixin_ = True @@ -1336,8 +1270,7 @@ class OpAssembler(IntOpAssembler, GuardOpAssembler, MiscOpAssembler, FieldOpAssembler, - StrOpAssembler, CallOpAssembler, - UnicodeOpAssembler, ForceOpAssembler, + CallOpAssembler, ForceOpAssembler, AllocOpAssembler, FloatOpAssembler, VectorAssembler): _mixin_ = True diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -802,18 +802,6 @@ temp_loc = r.SCRATCH2 return [base_loc, temp_loc] - def prepare_copystrcontent(self, op): - src_ptr_loc = self.ensure_reg(op.getarg(0)) - dst_ptr_loc = self.ensure_reg(op.getarg(1)) - src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) - dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) - length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - self._spill_before_call(gc_level=0) - return [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] - - prepare_copyunicodecontent = prepare_copystrcontent - prepare_same_as_i = helper.prepare_unary_op prepare_same_as_r = helper.prepare_unary_op prepare_same_as_f = helper.prepare_unary_op diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1222,78 +1222,16 @@ resloc = self.force_allocate_reg(op, [op.getarg(0)]) self.perform(op, [argloc], resloc) - def consider_copystrcontent(self, op): - self._consider_copystrcontent(op, is_unicode=False) - - def consider_copyunicodecontent(self, op): - self._consider_copystrcontent(op, is_unicode=True) - - def _consider_copystrcontent(self, op, is_unicode): - # compute the source address - args = op.getarglist() - base_loc = self.rm.make_sure_var_in_reg(args[0], args) - ofs_loc = self.rm.make_sure_var_in_reg(args[2], args) - assert args[0] is not args[1] # forbidden case of aliasing - srcaddr_box = TempVar() - forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = self.rm.force_allocate_reg(srcaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, - is_unicode=is_unicode) - # compute the destination address - base_loc = self.rm.make_sure_var_in_reg(args[1], forbidden_vars) - ofs_loc = self.rm.make_sure_var_in_reg(args[3], forbidden_vars) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempVar() - dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, - is_unicode=is_unicode) - # compute the length in bytes - length_box = args[4] - length_loc = self.loc(length_box) - if is_unicode: - forbidden_vars = [srcaddr_box, dstaddr_box] - bytes_box = TempVar() - bytes_loc = self.rm.force_allocate_reg(bytes_box, forbidden_vars) - scale = self._get_unicode_item_scale() - if not (isinstance(length_loc, ImmedLoc) or - isinstance(length_loc, RegLoc)): - self.assembler.mov(length_loc, bytes_loc) - length_loc = bytes_loc - self.assembler.load_effective_addr(length_loc, 0, scale, bytes_loc) - length_box = bytes_box - length_loc = bytes_loc - # call memcpy() - self.rm.before_call() - self.xrm.before_call() - self.assembler.simple_call_no_collect(imm(self.assembler.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) - self.rm.possibly_free_var(length_box) - self.rm.possibly_free_var(dstaddr_box) - self.rm.possibly_free_var(srcaddr_box) - - def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): - if is_unicode: - ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - scale = self._get_unicode_item_scale() - else: - ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.translate_support_code) - assert itemsize == 1 - ofs_items -= 1 # for the extra null character - scale = 0 - self.assembler.load_effective_addr(ofsloc, ofs_items, scale, - resloc, baseloc) - - def _get_unicode_item_scale(self): - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") + def consider_load_effective_address(self, op): + p0 = op.getarg(0) + i0 = op.getarg(1) + ploc = self.make_sure_var_in_reg(p0, [i0]) + iloc = self.make_sure_var_in_reg(i0, [p0]) + res = self.rm.force_allocate_reg(op, [p0, i0]) + assert isinstance(op.getarg(2), ConstInt) + assert isinstance(op.getarg(3), ConstInt) + self.assembler.load_effective_addr(iloc, op.getarg(2).getint(), + op.getarg(3).getint(), res, ploc) def _consider_math_read_timestamp(self, op): # hint: try to move unrelated registers away from eax and edx now diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -16,6 +16,7 @@ debug = True supports_floats = True supports_singlefloats = True + supports_load_effective_address = True dont_keepalive_stuff = False # for tests with_threads = False diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -963,75 +963,15 @@ def _mem_offset_supported(self, value): return -2**19 <= value < 2**19 - def emit_copystrcontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=False) - - def emit_copyunicodecontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=True) - - def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale): - if src_ofs.is_imm(): - value = src_ofs.value << scale - if check_imm_value(value): - self.mc.AGHIK(dst, src_ptr, l.imm(value)) - else: - # it is fine to use r1 here, because it will - # only hold a value before invoking the memory copy - self.mc.load_imm(r.SCRATCH, value) - self.mc.AGRK(dst, src_ptr, r.SCRATCH) - elif scale == 0: - self.mc.AGRK(dst, src_ptr, src_ofs) - else: - self.mc.SLLG(r.SCRATCH, src_ofs, l.addr(scale)) - self.mc.AGRK(dst, src_ptr, r.SCRATCH) - - def _emit_copycontent(self, arglocs, is_unicode): - [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] = arglocs - - if is_unicode: - basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 2: scale = 1 - elif itemsize == 4: scale = 2 - else: raise AssertionError - else: - basesize, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - basesize -= 1 # for the extra null character - scale = 0 - - # src and src_len are tmp registers - src = src_ptr_loc - src_len = r.odd_reg(src) - dst = r.r0 - dst_len = r.r1 - self._emit_load_for_copycontent(src, src_ptr_loc, src_ofs_loc, scale) - self._emit_load_for_copycontent(dst, dst_ptr_loc, dst_ofs_loc, scale) - - if length_loc.is_imm(): - length = length_loc.getint() - self.mc.load_imm(dst_len, length << scale) - else: - if scale > 0: - self.mc.SLLG(dst_len, length_loc, l.addr(scale)) - else: - self.mc.LGR(dst_len, length_loc) - # ensure that src_len is as long as dst_len, otherwise - # padding bytes are written to dst - self.mc.LGR(src_len, dst_len) - - self.mc.AGHI(src, l.imm(basesize)) - self.mc.AGHI(dst, l.imm(basesize)) - - # s390x has memset directly as a hardware instruction!! - # 0xB8 means we might reference dst later - self.mc.MVCLE(dst, src, l.addr(0xB8)) - # NOTE this instruction can (determined by the cpu), just - # quit the movement any time, thus it is looped until all bytes - # are copied! - self.mc.BRC(c.OF, l.imm(-self.mc.MVCLE_byte_count)) + # ...copystrcontent logic was removed, but note that + # if we want to reintroduce support for that: + # s390x has memset directly as a hardware instruction!! + # 0xB8 means we might reference dst later + #self.mc.MVCLE(dst, src, l.addr(0xB8)) + # NOTE this instruction can (determined by the cpu), just + # quit the movement any time, thus it is looped until all bytes + # are copied! + #self.mc.BRC(c.OF, l.imm(-self.mc.MVCLE_byte_count)) def emit_zero_array(self, op, arglocs, regalloc): base_loc, startindex_loc, length_loc, \ diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py --- a/rpython/jit/backend/zarch/regalloc.py +++ b/rpython/jit/backend/zarch/regalloc.py @@ -1269,29 +1269,6 @@ loc1 = self.ensure_reg(op.getarg(1)) return [loc0, loc1] - def prepare_copystrcontent(self, op): - """ this function needs five registers. - src & src_len: are allocated using ensure_even_odd_pair. - note that these are tmp registers, thus the actual variable - value is not modified. - src_len: when entering the assembler, src_ofs_loc's value is contained - in src_len register. - """ - src_ptr_loc, _ = \ - self.rm.ensure_even_odd_pair(op.getarg(0), - None, bind_first=True, - must_exist=False, load_loc_odd=False) - src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) - dst_ptr_loc = self.ensure_reg(op.getarg(1)) - dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) - length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - # no need to spill, we do not call memcpy, but we use s390x's - # hardware instruction to copy memory - return [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] - - prepare_copyunicodecontent = prepare_copystrcontent - def prepare_label(self, op): descr = op.getdescr() assert isinstance(descr, TargetToken) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -441,6 +441,7 @@ rop.GC_STORE, rop.GC_STORE_INDEXED, rop.LOAD_FROM_GC_TABLE, + rop.LOAD_EFFECTIVE_ADDRESS, ): # list of opcodes never executed by pyjitpl continue if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST: diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1055,6 +1055,8 @@ 'UNICODEGETITEM/2/i', # 'LOAD_FROM_GC_TABLE/1/r', # only emitted by rewrite.py + 'LOAD_EFFECTIVE_ADDRESS/4/i', # only emitted by rewrite.py, only if + # cpu.supports_load_effective_address. [v_gcptr,v_index,c_baseofs,c_shift] # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- From pypy.commits at gmail.com Sun Jun 9 14:33:51 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 09 Jun 2019 11:33:51 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <5cfd510f.1c69fb81.5cb9b.9659@mx.google.com> Author: Matti Picus Branch: Changeset: r96784:70b04c922db4 Date: 2019-06-09 21:33 +0300 http://bitbucket.org/pypy/pypy/changeset/70b04c922db4/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -38,3 +38,10 @@ .. branch: optimizeopt-cleanup Cleanup optimizeopt + +.. branch: copystrcontents-in-rewrite + +Remove ``copystrcontent`` and ``copyunicodecontent`` in the backends. +Instead, replace it in ``rewrite.py`` with a direct call to ``memcpy()`` and +new basic operation, ``load_effective_address``, which the backend can +even decide not to implement. From pypy.commits at gmail.com Tue Jun 11 01:44:03 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 10 Jun 2019 22:44:03 -0700 (PDT) Subject: [pypy-commit] pypy default: set MACOSX_DEPLOYMENT_TARGET for darwin, issue 2994 Message-ID: <5cff3fa3.1c69fb81.7496b.de75@mx.google.com> Author: Matti Picus Branch: Changeset: r96785:3bcc9f863e44 Date: 2019-06-10 21:37 +0300 http://bitbucket.org/pypy/pypy/changeset/3bcc9f863e44/ Log: set MACOSX_DEPLOYMENT_TARGET for darwin, issue 2994 diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -86,6 +86,7 @@ arch = platform.machine() g['LDSHARED'] += ' -undefined dynamic_lookup' g['CC'] += ' -arch %s' % (arch,) + g['MACOSX_DEPLOYMENT_TARGET'] = 10.14 global _config_vars _config_vars = g From pypy.commits at gmail.com Tue Jun 11 01:44:06 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 10 Jun 2019 22:44:06 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge default into py3.6 Message-ID: <5cff3fa6.1c69fb81.8a038.ea13@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96786:d2e310118a69 Date: 2019-06-11 08:37 +0300 http://bitbucket.org/pypy/pypy/changeset/d2e310118a69/ Log: merge default into py3.6 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -42,3 +42,10 @@ .. branch: optimizeopt-cleanup Cleanup optimizeopt + +.. branch: copystrcontents-in-rewrite + +Remove ``copystrcontent`` and ``copyunicodecontent`` in the backends. +Instead, replace it in ``rewrite.py`` with a direct call to ``memcpy()`` and +new basic operation, ``load_effective_address``, which the backend can +even decide not to implement. diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -834,73 +834,11 @@ else: assert 0 - #from ../x86/regalloc.py:928 ff. - def emit_op_copystrcontent(self, op, arglocs, regalloc, fcond): - assert len(arglocs) == 0 - self._emit_copystrcontent(op, regalloc, fcond, is_unicode=False) + def emit_op_load_effective_address(self, op, arglocs, regalloc, fcond): + self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[3].value, + arglocs[2].value) return fcond - def emit_op_copyunicodecontent(self, op, arglocs, regalloc, fcond): - assert len(arglocs) == 0 - self._emit_copystrcontent(op, regalloc, fcond, is_unicode=True) - return fcond - - def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): - # compute the source address - args = op.getarglist() - base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) - ofs_loc = regalloc.rm.make_sure_var_in_reg(args[2], args) - assert args[0] is not args[1] # forbidden case of aliasing - srcaddr_box = TempVar() - forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = regalloc.rm.force_allocate_reg(srcaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, - is_unicode=is_unicode) - # compute the destination address - base_loc = regalloc.rm.make_sure_var_in_reg(args[1], forbidden_vars) - ofs_loc = regalloc.rm.make_sure_var_in_reg(args[3], forbidden_vars) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempVar() - dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, - is_unicode=is_unicode) - # compute the length in bytes - length_box = args[4] - length_loc = regalloc.loc(length_box) - if is_unicode: - forbidden_vars = [srcaddr_box, dstaddr_box] - bytes_box = TempVar() - bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, forbidden_vars) - scale = self._get_unicode_item_scale() - if not length_loc.is_core_reg(): - self.regalloc_mov(length_loc, bytes_loc) - length_loc = bytes_loc - assert length_loc.is_core_reg() - self.mc.MOV_ri(r.ip.value, 1 << scale) - self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) - length_box = bytes_box - length_loc = bytes_loc - # call memcpy() - regalloc.before_call() - self.simple_call_no_collect(imm(self.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) - regalloc.rm.possibly_free_var(length_box) - regalloc.rm.possibly_free_var(dstaddr_box) - regalloc.rm.possibly_free_var(srcaddr_box) - - def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): - if is_unicode: - ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - scale = self._get_unicode_item_scale() - else: - ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - ofs_items -= 1 # for the extra null character - scale = 0 - self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items) - # result = base_loc + (scaled_loc << scale) + static_offset def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0): assert scaled_loc.is_core_reg() @@ -915,16 +853,6 @@ self.mc.ADD_rr(result.value, base_loc.value, scaled_loc.value) self.mc.ADD_ri(result.value, result.value, static_offset) - def _get_unicode_item_scale(self): - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") - def store_force_descr(self, op, fail_locs, frame_depth): pos = self.mc.currpos() guard_token = self.build_guard_token(op, frame_depth, fail_locs, pos, c.AL) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -873,8 +873,6 @@ prepare_op_gc_load_indexed_r = _prepare_op_gc_load_indexed prepare_op_gc_load_indexed_f = _prepare_op_gc_load_indexed - prepare_op_copystrcontent = void - prepare_op_copyunicodecontent = void prepare_op_zero_array = void def _prepare_op_same_as(self, op, fcond): @@ -899,6 +897,13 @@ resloc = self.force_allocate_reg(op) return [resloc] + def prepare_op_load_effective_address(self, op, fcond): + args = op.getarglist() + arg0 = self.make_sure_var_in_reg(args[0], args) + arg1 = self.make_sure_var_in_reg(args[1], args) + res = self.force_allocate_reg(op) + return [arg0, arg1, args[2], args[3], res] + def prepare_op_call_malloc_nursery(self, op, fcond): size_box = op.getarg(0) assert isinstance(size_box, ConstInt) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -23,6 +23,7 @@ supports_floats = True supports_longlong = True supports_singlefloats = True + supports_load_effective_address = True from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE all_reg_indexes = range(len(all_regs)) diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -14,6 +14,7 @@ from rpython.jit.metainterp.support import ptr2int from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.symbolic import WORD +from rpython.jit.backend.llsupport.memcpy import memcpy_fn from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr, FieldDescr from rpython.jit.backend.llsupport.descr import GcCache, get_field_descr from rpython.jit.backend.llsupport.descr import get_array_descr @@ -36,6 +37,11 @@ self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, 'typeptr') self._generated_functions = [] + self.memcpy_fn = memcpy_fn + self.memcpy_descr = get_call_descr(self, + [lltype.Signed, lltype.Signed, lltype.Signed], lltype.Void, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, + can_collect=False)) def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -34,6 +34,10 @@ - Add COND_CALLs to the write barrier before SETFIELD_GC and SETARRAYITEM_GC operations. + - Rewrites copystrcontent to a call to memcopy + + - XXX does more than that, please write it down + '_write_barrier_applied' contains a dictionary of variable -> None. If a variable is in the dictionary, next setfields can be called without a write barrier. The idea is that an object that was freshly allocated @@ -335,6 +339,10 @@ self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: self.emit_label() + # ---- change COPY{STR|UNICODE}CONTENT into a call ------ + if op.opnum == rop.COPYSTRCONTENT or op.opnum == rop.COPYUNICODECONTENT: + self.rewrite_copy_str_content(op) + continue # ---------- write barriers ---------- if self.gc_ll_descr.write_barrier_descr is not None: if op.getopnum() == rop.SETFIELD_GC: @@ -953,6 +961,61 @@ self.gcrefs_output_list.append(gcref) return index + def rewrite_copy_str_content(self, op): + funcaddr = llmemory.cast_ptr_to_adr(self.gc_ll_descr.memcpy_fn) + memcpy_fn = self.cpu.cast_adr_to_int(funcaddr) + memcpy_descr = self.gc_ll_descr.memcpy_descr + if op.getopnum() == rop.COPYSTRCONTENT: + basesize = self.gc_ll_descr.str_descr.basesize + # because we have one extra item after alloc, the actual address + # of string start is 1 lower, from extra_item_after_malloc + basesize -= 1 + assert self.gc_ll_descr.str_descr.itemsize == 1 + itemscale = 0 + else: + basesize = self.gc_ll_descr.unicode_descr.basesize + itemsize = self.gc_ll_descr.unicode_descr.itemsize + if itemsize == 2: + itemscale = 1 + elif itemsize == 4: + itemscale = 2 + else: + assert False, "unknown size of unicode" + i1 = self.emit_load_effective_address(op.getarg(0), op.getarg(2), + basesize, itemscale) + i2 = self.emit_load_effective_address(op.getarg(1), op.getarg(3), + basesize, itemscale) + if op.getopnum() == rop.COPYSTRCONTENT: + arg = op.getarg(4) + else: + # do some basic constant folding + if isinstance(op.getarg(4), ConstInt): + arg = ConstInt(op.getarg(4).getint() << itemscale) + else: + arg = ResOperation(rop.INT_LSHIFT, + [op.getarg(4), ConstInt(itemscale)]) + self.emit_op(arg) + self.emit_op(ResOperation(rop.CALL_N, + [ConstInt(memcpy_fn), i2, i1, arg], descr=memcpy_descr)) + + def emit_load_effective_address(self, v_gcptr, v_index, base, itemscale): + if self.cpu.supports_load_effective_address: + i1 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS, + [v_gcptr, v_index, ConstInt(base), + ConstInt(itemscale)]) + self.emit_op(i1) + return i1 + else: + if itemscale > 0: + v_index = ResOperation(rop.INT_LSHIFT, + [v_index, ConstInt(itemscale)]) + self.emit_op(v_index) + i1b = ResOperation(rop.INT_ADD, [v_gcptr, v_index]) + self.emit_op(i1b) + i1 = ResOperation(rop.INT_ADD, [i1b, ConstInt(base)]) + self.emit_op(i1) + return i1 + def remove_constptr(self, c): """Remove all ConstPtrs, and replace them with load_from_gc_table. """ diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -142,11 +142,16 @@ raw_sfdescr = get_array_descr(self.gc_ll_descr, RAW_SF) # strdescr = self.gc_ll_descr.str_descr + str_basesize = self.gc_ll_descr.str_descr.basesize - 1 unicodedescr = self.gc_ll_descr.unicode_descr strlendescr = strdescr.lendescr unicodelendescr = unicodedescr.lendescr strhashdescr = self.gc_ll_descr.str_hash_descr unicodehashdescr = self.gc_ll_descr.unicode_hash_descr + uni_basesize = unicodedescr.basesize + uni_itemscale = {2: 1, 4: 2}[unicodedescr.itemsize] + memcpy_fn = self.gc_ll_descr.memcpy_fn + memcpy_descr = self.gc_ll_descr.memcpy_descr casmdescr = JitCellToken() clt = FakeLoopToken() @@ -169,6 +174,7 @@ signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr casmdescr.compiled_loop_token = clt + # guarddescr = AbstractFailDescr() # @@ -200,6 +206,7 @@ load_constant_offset = True load_supported_factors = (1,2,4,8) + supports_load_effective_address = True translate_support_code = None @@ -237,6 +244,9 @@ self._cache[key] = r return r + def cast_adr_to_int(self, adr): + return llmemory.AddressAsInt(adr) + class TestBoehm(RewriteTests): def setup_method(self, meth): class FakeCPU(BaseFakeCPU): @@ -1436,3 +1446,57 @@ jump() """) assert len(self.gcrefs) == 2 + + def test_rewrite_copystrcontents(self): + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copystrcontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2 = load_effective_address(p0, i0, %(str_basesize)s, 0) + i3 = load_effective_address(p1, i1, %(str_basesize)s, 0) + call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) + """) + + def test_rewrite_copystrcontents_without_load_effective_address(self): + self.cpu.supports_load_effective_address = False + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copystrcontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2b = int_add(p0, i0) + i2 = int_add(i2b, %(str_basesize)s) + i3b = int_add(p1, i1) + i3 = int_add(i3b, %(str_basesize)s) + call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) + """) + + def test_rewrite_copyunicodecontents(self): + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copyunicodecontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2 = load_effective_address(p0, i0, %(uni_basesize)s, %(uni_itemscale)d) + i3 = load_effective_address(p1, i1, %(uni_basesize)s, %(uni_itemscale)d) + i4 = int_lshift(i_len, %(uni_itemscale)d) + call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr) + """) + + def test_rewrite_copyunicodecontents_without_load_effective_address(self): + self.cpu.supports_load_effective_address = False + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copyunicodecontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i0s = int_lshift(i0, %(uni_itemscale)d) + i2b = int_add(p0, i0s) + i2 = int_add(i2b, %(uni_basesize)s) + i1s = int_lshift(i1, %(uni_itemscale)d) + i3b = int_add(p1, i1s) + i3 = int_add(i3b, %(uni_basesize)s) + i4 = int_lshift(i_len, %(uni_itemscale)d) + call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr) + """) diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -19,6 +19,7 @@ # Boxes and Consts are BoxFloats and ConstFloats. supports_singlefloats = False supports_guard_gc_type = False + supports_load_effective_address = False propagate_exception_descr = None diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -966,72 +966,6 @@ pmc.overwrite() -class StrOpAssembler(object): - - _mixin_ = True - - def emit_copystrcontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=False) - - def emit_copyunicodecontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=True) - - def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale): - if src_ofs.is_imm(): - value = src_ofs.value << scale - if value < 32768: - self.mc.addi(dst.value, src_ptr.value, value) - else: - self.mc.load_imm(dst, value) - self.mc.add(dst.value, src_ptr.value, dst.value) - elif scale == 0: - self.mc.add(dst.value, src_ptr.value, src_ofs.value) - else: - self.mc.sldi(dst.value, src_ofs.value, scale) - self.mc.add(dst.value, src_ptr.value, dst.value) - - def _emit_copycontent(self, arglocs, is_unicode): - [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] = arglocs - - if is_unicode: - basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 2: scale = 1 - elif itemsize == 4: scale = 2 - else: raise AssertionError - else: - basesize, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - basesize -= 1 # for the extra null character - scale = 0 - - self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale) - self._emit_load_for_copycontent(r.r2, dst_ptr_loc, dst_ofs_loc, scale) - - if length_loc.is_imm(): - length = length_loc.getint() - self.mc.load_imm(r.r5, length << scale) - else: - if scale > 0: - self.mc.sldi(r.r5.value, length_loc.value, scale) - elif length_loc is not r.r5: - self.mc.mr(r.r5.value, length_loc.value) - - self.mc.mr(r.r4.value, r.r0.value) - self.mc.addi(r.r4.value, r.r4.value, basesize) - self.mc.addi(r.r3.value, r.r2.value, basesize) - - self.mc.load_imm(self.mc.RAW_CALL_REG, self.memcpy_addr) - self.mc.raw_call() - - -class UnicodeOpAssembler(object): - _mixin_ = True - # empty! - - class AllocOpAssembler(object): _mixin_ = True @@ -1336,8 +1270,7 @@ class OpAssembler(IntOpAssembler, GuardOpAssembler, MiscOpAssembler, FieldOpAssembler, - StrOpAssembler, CallOpAssembler, - UnicodeOpAssembler, ForceOpAssembler, + CallOpAssembler, ForceOpAssembler, AllocOpAssembler, FloatOpAssembler, VectorAssembler): _mixin_ = True diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -802,18 +802,6 @@ temp_loc = r.SCRATCH2 return [base_loc, temp_loc] - def prepare_copystrcontent(self, op): - src_ptr_loc = self.ensure_reg(op.getarg(0)) - dst_ptr_loc = self.ensure_reg(op.getarg(1)) - src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) - dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) - length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - self._spill_before_call(gc_level=0) - return [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] - - prepare_copyunicodecontent = prepare_copystrcontent - prepare_same_as_i = helper.prepare_unary_op prepare_same_as_r = helper.prepare_unary_op prepare_same_as_f = helper.prepare_unary_op diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1222,78 +1222,16 @@ resloc = self.force_allocate_reg(op, [op.getarg(0)]) self.perform(op, [argloc], resloc) - def consider_copystrcontent(self, op): - self._consider_copystrcontent(op, is_unicode=False) - - def consider_copyunicodecontent(self, op): - self._consider_copystrcontent(op, is_unicode=True) - - def _consider_copystrcontent(self, op, is_unicode): - # compute the source address - args = op.getarglist() - base_loc = self.rm.make_sure_var_in_reg(args[0], args) - ofs_loc = self.rm.make_sure_var_in_reg(args[2], args) - assert args[0] is not args[1] # forbidden case of aliasing - srcaddr_box = TempVar() - forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = self.rm.force_allocate_reg(srcaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, - is_unicode=is_unicode) - # compute the destination address - base_loc = self.rm.make_sure_var_in_reg(args[1], forbidden_vars) - ofs_loc = self.rm.make_sure_var_in_reg(args[3], forbidden_vars) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempVar() - dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, - is_unicode=is_unicode) - # compute the length in bytes - length_box = args[4] - length_loc = self.loc(length_box) - if is_unicode: - forbidden_vars = [srcaddr_box, dstaddr_box] - bytes_box = TempVar() - bytes_loc = self.rm.force_allocate_reg(bytes_box, forbidden_vars) - scale = self._get_unicode_item_scale() - if not (isinstance(length_loc, ImmedLoc) or - isinstance(length_loc, RegLoc)): - self.assembler.mov(length_loc, bytes_loc) - length_loc = bytes_loc - self.assembler.load_effective_addr(length_loc, 0, scale, bytes_loc) - length_box = bytes_box - length_loc = bytes_loc - # call memcpy() - self.rm.before_call() - self.xrm.before_call() - self.assembler.simple_call_no_collect(imm(self.assembler.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) - self.rm.possibly_free_var(length_box) - self.rm.possibly_free_var(dstaddr_box) - self.rm.possibly_free_var(srcaddr_box) - - def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): - if is_unicode: - ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - scale = self._get_unicode_item_scale() - else: - ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.translate_support_code) - assert itemsize == 1 - ofs_items -= 1 # for the extra null character - scale = 0 - self.assembler.load_effective_addr(ofsloc, ofs_items, scale, - resloc, baseloc) - - def _get_unicode_item_scale(self): - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") + def consider_load_effective_address(self, op): + p0 = op.getarg(0) + i0 = op.getarg(1) + ploc = self.make_sure_var_in_reg(p0, [i0]) + iloc = self.make_sure_var_in_reg(i0, [p0]) + res = self.rm.force_allocate_reg(op, [p0, i0]) + assert isinstance(op.getarg(2), ConstInt) + assert isinstance(op.getarg(3), ConstInt) + self.assembler.load_effective_addr(iloc, op.getarg(2).getint(), + op.getarg(3).getint(), res, ploc) def _consider_math_read_timestamp(self, op): # hint: try to move unrelated registers away from eax and edx now diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -16,6 +16,7 @@ debug = True supports_floats = True supports_singlefloats = True + supports_load_effective_address = True dont_keepalive_stuff = False # for tests with_threads = False diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -963,75 +963,15 @@ def _mem_offset_supported(self, value): return -2**19 <= value < 2**19 - def emit_copystrcontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=False) - - def emit_copyunicodecontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=True) - - def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale): - if src_ofs.is_imm(): - value = src_ofs.value << scale - if check_imm_value(value): - self.mc.AGHIK(dst, src_ptr, l.imm(value)) - else: - # it is fine to use r1 here, because it will - # only hold a value before invoking the memory copy - self.mc.load_imm(r.SCRATCH, value) - self.mc.AGRK(dst, src_ptr, r.SCRATCH) - elif scale == 0: - self.mc.AGRK(dst, src_ptr, src_ofs) - else: - self.mc.SLLG(r.SCRATCH, src_ofs, l.addr(scale)) - self.mc.AGRK(dst, src_ptr, r.SCRATCH) - - def _emit_copycontent(self, arglocs, is_unicode): - [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] = arglocs - - if is_unicode: - basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 2: scale = 1 - elif itemsize == 4: scale = 2 - else: raise AssertionError - else: - basesize, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - basesize -= 1 # for the extra null character - scale = 0 - - # src and src_len are tmp registers - src = src_ptr_loc - src_len = r.odd_reg(src) - dst = r.r0 - dst_len = r.r1 - self._emit_load_for_copycontent(src, src_ptr_loc, src_ofs_loc, scale) - self._emit_load_for_copycontent(dst, dst_ptr_loc, dst_ofs_loc, scale) - - if length_loc.is_imm(): - length = length_loc.getint() - self.mc.load_imm(dst_len, length << scale) - else: - if scale > 0: - self.mc.SLLG(dst_len, length_loc, l.addr(scale)) - else: - self.mc.LGR(dst_len, length_loc) - # ensure that src_len is as long as dst_len, otherwise - # padding bytes are written to dst - self.mc.LGR(src_len, dst_len) - - self.mc.AGHI(src, l.imm(basesize)) - self.mc.AGHI(dst, l.imm(basesize)) - - # s390x has memset directly as a hardware instruction!! - # 0xB8 means we might reference dst later - self.mc.MVCLE(dst, src, l.addr(0xB8)) - # NOTE this instruction can (determined by the cpu), just - # quit the movement any time, thus it is looped until all bytes - # are copied! - self.mc.BRC(c.OF, l.imm(-self.mc.MVCLE_byte_count)) + # ...copystrcontent logic was removed, but note that + # if we want to reintroduce support for that: + # s390x has memset directly as a hardware instruction!! + # 0xB8 means we might reference dst later + #self.mc.MVCLE(dst, src, l.addr(0xB8)) + # NOTE this instruction can (determined by the cpu), just + # quit the movement any time, thus it is looped until all bytes + # are copied! + #self.mc.BRC(c.OF, l.imm(-self.mc.MVCLE_byte_count)) def emit_zero_array(self, op, arglocs, regalloc): base_loc, startindex_loc, length_loc, \ diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py --- a/rpython/jit/backend/zarch/regalloc.py +++ b/rpython/jit/backend/zarch/regalloc.py @@ -1269,29 +1269,6 @@ loc1 = self.ensure_reg(op.getarg(1)) return [loc0, loc1] - def prepare_copystrcontent(self, op): - """ this function needs five registers. - src & src_len: are allocated using ensure_even_odd_pair. - note that these are tmp registers, thus the actual variable - value is not modified. - src_len: when entering the assembler, src_ofs_loc's value is contained - in src_len register. - """ - src_ptr_loc, _ = \ - self.rm.ensure_even_odd_pair(op.getarg(0), - None, bind_first=True, - must_exist=False, load_loc_odd=False) - src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) - dst_ptr_loc = self.ensure_reg(op.getarg(1)) - dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) - length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - # no need to spill, we do not call memcpy, but we use s390x's - # hardware instruction to copy memory - return [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] - - prepare_copyunicodecontent = prepare_copystrcontent - def prepare_label(self, op): descr = op.getdescr() assert isinstance(descr, TargetToken) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -441,6 +441,7 @@ rop.GC_STORE, rop.GC_STORE_INDEXED, rop.LOAD_FROM_GC_TABLE, + rop.LOAD_EFFECTIVE_ADDRESS, ): # list of opcodes never executed by pyjitpl continue if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST: diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -629,6 +629,20 @@ # and then emit the operation return self.emit(op) + def optimize_GC_LOAD_I(self, op): + # seeing a 'gc_load*' forces all the lazy sets that are still + # pending, as an approximation. We could try to be really clever + # and only force some of them, but we don't have any descr here. + self.force_all_lazy_sets() + self.make_nonnull(op.getarg(0)) + return self.emit(op) + optimize_GC_LOAD_R = optimize_GC_LOAD_I + optimize_GC_LOAD_F = optimize_GC_LOAD_I + + optimize_GC_LOAD_INDEXED_I = optimize_GC_LOAD_I + optimize_GC_LOAD_INDEXED_R = optimize_GC_LOAD_I + optimize_GC_LOAD_INDEXED_F = optimize_GC_LOAD_I + def optimize_QUASIIMMUT_FIELD(self, op): # Pattern: QUASIIMMUT_FIELD(s, descr=QuasiImmutDescr) # x = GETFIELD_GC(s, descr='inst_x') # pure diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -9495,3 +9495,42 @@ jump(i3, i2, i3) """ self.optimize_loop(ops, expected) + + def test_issue3014(self): + # 'gc_load_indexed' must force 'setarrayitem_gc' + ops = """ + [i183] + p0 = new_array(5, descr=arraydescr) + setarrayitem_gc(p0, 0, i183, descr=arraydescr) + i235 = gc_load_indexed_i(p0, 0, 1, 16, 2) + escape_i(i235) + jump(i183) + """ + self.optimize_loop(ops, ops) + + def test_issue3014_2(self): + # same rules for gc_store_indexed versus getarrayitem_gc, + # and 'gc_store_indexed' invalidates the value for 'getarrayitem_gc' + # (in this direction it seems to work already) + ops = """ + [p0, i183] + i234 = getarrayitem_gc_i(p0, 0, descr=arraydescr) + gc_store_indexed(p0, 0, i183, 1, 16, 2) + i235 = getarrayitem_gc_i(p0, 0, descr=arraydescr) + escape_i(i234) + escape_i(i235) + jump(p0, i183) + """ + self.optimize_loop(ops, ops) + + def test_issue3014_3(self): + # 'gc_load' must force 'setfield_gc' + ops = """ + [i183] + p0 = new(descr=ssize) + setfield_gc(p0, i183, descr=adescr) + i235 = gc_load_i(p0, 8, 2) + escape_i(i235) + jump(i183) + """ + self.optimize_loop(ops, ops) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1055,6 +1055,8 @@ 'UNICODEGETITEM/2/i', # 'LOAD_FROM_GC_TABLE/1/r', # only emitted by rewrite.py + 'LOAD_EFFECTIVE_ADDRESS/4/i', # only emitted by rewrite.py, only if + # cpu.supports_load_effective_address. [v_gcptr,v_index,c_baseofs,c_shift] # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- diff --git a/rpython/rtyper/test/test_rfloat.py b/rpython/rtyper/test/test_rfloat.py --- a/rpython/rtyper/test/test_rfloat.py +++ b/rpython/rtyper/test/test_rfloat.py @@ -253,3 +253,10 @@ return compute_hash(f) res = self.interpret(fn, [1.5]) assert res == compute_hash(1.5) + + def test_float_constant_inf(self): + from rpython.rlib.rfloat import INFINITY + def fn(): + return INFINITY # float('inf') is not supported by RPython so far + res = self.interpret(fn, []) + assert res == float('inf') From pypy.commits at gmail.com Tue Jun 11 01:44:07 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 10 Jun 2019 22:44:07 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: set minimum supported MACOSX_DEPLOYMENT_TARGET (like in 3bcc9f863e44) Message-ID: <5cff3fa7.1c69fb81.345cb.cae7@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96787:71345b967667 Date: 2019-06-11 08:43 +0300 http://bitbucket.org/pypy/pypy/changeset/71345b967667/ Log: set minimum supported MACOSX_DEPLOYMENT_TARGET (like in 3bcc9f863e44) diff --git a/lib-python/3/_osx_support.py b/lib-python/3/_osx_support.py --- a/lib-python/3/_osx_support.py +++ b/lib-python/3/_osx_support.py @@ -107,7 +107,9 @@ if m is not None: _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2]) # else: fall back to the default behaviour - + if not _SYSTEM_VERSION: + # minimum supported MACOSX_DEPLOYMENT_TARGET version + return '10.14' return _SYSTEM_VERSION def _remove_original_values(_config_vars): From pypy.commits at gmail.com Tue Jun 11 07:21:30 2019 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 11 Jun 2019 04:21:30 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: skip if there is no hypothesis Message-ID: <5cff8eba.1c69fb81.1e6fe.380d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96788:028e00604df8 Date: 2019-06-11 11:55 +0200 http://bitbucket.org/pypy/pypy/changeset/028e00604df8/ Log: skip if there is no hypothesis diff --git a/pypy/module/_pypyjson/test/test_simd.py b/pypy/module/_pypyjson/test/test_simd.py --- a/pypy/module/_pypyjson/test/test_simd.py +++ b/pypy/module/_pypyjson/test/test_simd.py @@ -9,7 +9,10 @@ from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned, WORD_SIZE from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned_no_hash -from hypothesis import example, given, strategies +try: + from hypothesis import example, given, strategies +except ImportError: + pytest.skip("missing hypothesis!") if not USE_SIMD: pytest.skip("only implemented for 64 bit for now") From pypy.commits at gmail.com Tue Jun 11 07:21:32 2019 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 11 Jun 2019 04:21:32 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps: merge default Message-ID: <5cff8ebc.1c69fb81.553a9.2ce6@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps Changeset: r96789:126a8db4e19d Date: 2019-06-11 11:57 +0200 http://bitbucket.org/pypy/pypy/changeset/126a8db4e19d/ Log: merge default diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -86,6 +86,7 @@ arch = platform.machine() g['LDSHARED'] += ' -undefined dynamic_lookup' g['CC'] += ' -arch %s' % (arch,) + g['MACOSX_DEPLOYMENT_TARGET'] = 10.14 global _config_vars _config_vars = g diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -39,7 +39,15 @@ Cleanup optimizeopt +.. branch: copystrcontents-in-rewrite + +Remove ``copystrcontent`` and ``copyunicodecontent`` in the backends. +Instead, replace it in ``rewrite.py`` with a direct call to ``memcpy()`` and +new basic operation, ``load_effective_address``, which the backend can +even decide not to implement. + .. branch: json-decoder-maps Much faster and more memory-efficient JSON decoding. The resulting dictionaries that come out of the JSON decoder have faster lookups too. + diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -834,73 +834,11 @@ else: assert 0 - #from ../x86/regalloc.py:928 ff. - def emit_op_copystrcontent(self, op, arglocs, regalloc, fcond): - assert len(arglocs) == 0 - self._emit_copystrcontent(op, regalloc, fcond, is_unicode=False) + def emit_op_load_effective_address(self, op, arglocs, regalloc, fcond): + self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[3].value, + arglocs[2].value) return fcond - def emit_op_copyunicodecontent(self, op, arglocs, regalloc, fcond): - assert len(arglocs) == 0 - self._emit_copystrcontent(op, regalloc, fcond, is_unicode=True) - return fcond - - def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): - # compute the source address - args = op.getarglist() - base_loc = regalloc.rm.make_sure_var_in_reg(args[0], args) - ofs_loc = regalloc.rm.make_sure_var_in_reg(args[2], args) - assert args[0] is not args[1] # forbidden case of aliasing - srcaddr_box = TempVar() - forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = regalloc.rm.force_allocate_reg(srcaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, - is_unicode=is_unicode) - # compute the destination address - base_loc = regalloc.rm.make_sure_var_in_reg(args[1], forbidden_vars) - ofs_loc = regalloc.rm.make_sure_var_in_reg(args[3], forbidden_vars) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempVar() - dstaddr_loc = regalloc.rm.force_allocate_reg(dstaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, - is_unicode=is_unicode) - # compute the length in bytes - length_box = args[4] - length_loc = regalloc.loc(length_box) - if is_unicode: - forbidden_vars = [srcaddr_box, dstaddr_box] - bytes_box = TempVar() - bytes_loc = regalloc.rm.force_allocate_reg(bytes_box, forbidden_vars) - scale = self._get_unicode_item_scale() - if not length_loc.is_core_reg(): - self.regalloc_mov(length_loc, bytes_loc) - length_loc = bytes_loc - assert length_loc.is_core_reg() - self.mc.MOV_ri(r.ip.value, 1 << scale) - self.mc.MUL(bytes_loc.value, r.ip.value, length_loc.value) - length_box = bytes_box - length_loc = bytes_loc - # call memcpy() - regalloc.before_call() - self.simple_call_no_collect(imm(self.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) - regalloc.rm.possibly_free_var(length_box) - regalloc.rm.possibly_free_var(dstaddr_box) - regalloc.rm.possibly_free_var(srcaddr_box) - - def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): - if is_unicode: - ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - scale = self._get_unicode_item_scale() - else: - ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - ofs_items -= 1 # for the extra null character - scale = 0 - self._gen_address(resloc, baseloc, ofsloc, scale, ofs_items) - # result = base_loc + (scaled_loc << scale) + static_offset def _gen_address(self, result, base_loc, scaled_loc, scale=0, static_offset=0): assert scaled_loc.is_core_reg() @@ -915,16 +853,6 @@ self.mc.ADD_rr(result.value, base_loc.value, scaled_loc.value) self.mc.ADD_ri(result.value, result.value, static_offset) - def _get_unicode_item_scale(self): - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") - def store_force_descr(self, op, fail_locs, frame_depth): pos = self.mc.currpos() guard_token = self.build_guard_token(op, frame_depth, fail_locs, pos, c.AL) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -873,8 +873,6 @@ prepare_op_gc_load_indexed_r = _prepare_op_gc_load_indexed prepare_op_gc_load_indexed_f = _prepare_op_gc_load_indexed - prepare_op_copystrcontent = void - prepare_op_copyunicodecontent = void prepare_op_zero_array = void def _prepare_op_same_as(self, op, fcond): @@ -899,6 +897,13 @@ resloc = self.force_allocate_reg(op) return [resloc] + def prepare_op_load_effective_address(self, op, fcond): + args = op.getarglist() + arg0 = self.make_sure_var_in_reg(args[0], args) + arg1 = self.make_sure_var_in_reg(args[1], args) + res = self.force_allocate_reg(op) + return [arg0, arg1, args[2], args[3], res] + def prepare_op_call_malloc_nursery(self, op, fcond): size_box = op.getarg(0) assert isinstance(size_box, ConstInt) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -23,6 +23,7 @@ supports_floats = True supports_longlong = True supports_singlefloats = True + supports_load_effective_address = True from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE all_reg_indexes = range(len(all_regs)) diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -14,6 +14,7 @@ from rpython.jit.metainterp.support import ptr2int from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.symbolic import WORD +from rpython.jit.backend.llsupport.memcpy import memcpy_fn from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr, FieldDescr from rpython.jit.backend.llsupport.descr import GcCache, get_field_descr from rpython.jit.backend.llsupport.descr import get_array_descr @@ -36,6 +37,11 @@ self.fielddescr_vtable = get_field_descr(self, rclass.OBJECT, 'typeptr') self._generated_functions = [] + self.memcpy_fn = memcpy_fn + self.memcpy_descr = get_call_descr(self, + [lltype.Signed, lltype.Signed, lltype.Signed], lltype.Void, + EffectInfo([], [], [], [], [], [], EffectInfo.EF_CANNOT_RAISE, + can_collect=False)) def _setup_str(self): self.str_descr = get_array_descr(self, rstr.STR) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -34,6 +34,10 @@ - Add COND_CALLs to the write barrier before SETFIELD_GC and SETARRAYITEM_GC operations. + - Rewrites copystrcontent to a call to memcopy + + - XXX does more than that, please write it down + '_write_barrier_applied' contains a dictionary of variable -> None. If a variable is in the dictionary, next setfields can be called without a write barrier. The idea is that an object that was freshly allocated @@ -335,6 +339,10 @@ self.emitting_an_operation_that_can_collect() elif op.getopnum() == rop.LABEL: self.emit_label() + # ---- change COPY{STR|UNICODE}CONTENT into a call ------ + if op.opnum == rop.COPYSTRCONTENT or op.opnum == rop.COPYUNICODECONTENT: + self.rewrite_copy_str_content(op) + continue # ---------- write barriers ---------- if self.gc_ll_descr.write_barrier_descr is not None: if op.getopnum() == rop.SETFIELD_GC: @@ -953,6 +961,61 @@ self.gcrefs_output_list.append(gcref) return index + def rewrite_copy_str_content(self, op): + funcaddr = llmemory.cast_ptr_to_adr(self.gc_ll_descr.memcpy_fn) + memcpy_fn = self.cpu.cast_adr_to_int(funcaddr) + memcpy_descr = self.gc_ll_descr.memcpy_descr + if op.getopnum() == rop.COPYSTRCONTENT: + basesize = self.gc_ll_descr.str_descr.basesize + # because we have one extra item after alloc, the actual address + # of string start is 1 lower, from extra_item_after_malloc + basesize -= 1 + assert self.gc_ll_descr.str_descr.itemsize == 1 + itemscale = 0 + else: + basesize = self.gc_ll_descr.unicode_descr.basesize + itemsize = self.gc_ll_descr.unicode_descr.itemsize + if itemsize == 2: + itemscale = 1 + elif itemsize == 4: + itemscale = 2 + else: + assert False, "unknown size of unicode" + i1 = self.emit_load_effective_address(op.getarg(0), op.getarg(2), + basesize, itemscale) + i2 = self.emit_load_effective_address(op.getarg(1), op.getarg(3), + basesize, itemscale) + if op.getopnum() == rop.COPYSTRCONTENT: + arg = op.getarg(4) + else: + # do some basic constant folding + if isinstance(op.getarg(4), ConstInt): + arg = ConstInt(op.getarg(4).getint() << itemscale) + else: + arg = ResOperation(rop.INT_LSHIFT, + [op.getarg(4), ConstInt(itemscale)]) + self.emit_op(arg) + self.emit_op(ResOperation(rop.CALL_N, + [ConstInt(memcpy_fn), i2, i1, arg], descr=memcpy_descr)) + + def emit_load_effective_address(self, v_gcptr, v_index, base, itemscale): + if self.cpu.supports_load_effective_address: + i1 = ResOperation(rop.LOAD_EFFECTIVE_ADDRESS, + [v_gcptr, v_index, ConstInt(base), + ConstInt(itemscale)]) + self.emit_op(i1) + return i1 + else: + if itemscale > 0: + v_index = ResOperation(rop.INT_LSHIFT, + [v_index, ConstInt(itemscale)]) + self.emit_op(v_index) + i1b = ResOperation(rop.INT_ADD, [v_gcptr, v_index]) + self.emit_op(i1b) + i1 = ResOperation(rop.INT_ADD, [i1b, ConstInt(base)]) + self.emit_op(i1) + return i1 + def remove_constptr(self, c): """Remove all ConstPtrs, and replace them with load_from_gc_table. """ diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -142,11 +142,16 @@ raw_sfdescr = get_array_descr(self.gc_ll_descr, RAW_SF) # strdescr = self.gc_ll_descr.str_descr + str_basesize = self.gc_ll_descr.str_descr.basesize - 1 unicodedescr = self.gc_ll_descr.unicode_descr strlendescr = strdescr.lendescr unicodelendescr = unicodedescr.lendescr strhashdescr = self.gc_ll_descr.str_hash_descr unicodehashdescr = self.gc_ll_descr.unicode_hash_descr + uni_basesize = unicodedescr.basesize + uni_itemscale = {2: 1, 4: 2}[unicodedescr.itemsize] + memcpy_fn = self.gc_ll_descr.memcpy_fn + memcpy_descr = self.gc_ll_descr.memcpy_descr casmdescr = JitCellToken() clt = FakeLoopToken() @@ -169,6 +174,7 @@ signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr casmdescr.compiled_loop_token = clt + # guarddescr = AbstractFailDescr() # @@ -200,6 +206,7 @@ load_constant_offset = True load_supported_factors = (1,2,4,8) + supports_load_effective_address = True translate_support_code = None @@ -237,6 +244,9 @@ self._cache[key] = r return r + def cast_adr_to_int(self, adr): + return llmemory.AddressAsInt(adr) + class TestBoehm(RewriteTests): def setup_method(self, meth): class FakeCPU(BaseFakeCPU): @@ -1436,3 +1446,57 @@ jump() """) assert len(self.gcrefs) == 2 + + def test_rewrite_copystrcontents(self): + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copystrcontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2 = load_effective_address(p0, i0, %(str_basesize)s, 0) + i3 = load_effective_address(p1, i1, %(str_basesize)s, 0) + call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) + """) + + def test_rewrite_copystrcontents_without_load_effective_address(self): + self.cpu.supports_load_effective_address = False + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copystrcontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2b = int_add(p0, i0) + i2 = int_add(i2b, %(str_basesize)s) + i3b = int_add(p1, i1) + i3 = int_add(i3b, %(str_basesize)s) + call_n(ConstClass(memcpy_fn), i3, i2, i_len, descr=memcpy_descr) + """) + + def test_rewrite_copyunicodecontents(self): + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copyunicodecontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i2 = load_effective_address(p0, i0, %(uni_basesize)s, %(uni_itemscale)d) + i3 = load_effective_address(p1, i1, %(uni_basesize)s, %(uni_itemscale)d) + i4 = int_lshift(i_len, %(uni_itemscale)d) + call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr) + """) + + def test_rewrite_copyunicodecontents_without_load_effective_address(self): + self.cpu.supports_load_effective_address = False + self.check_rewrite(""" + [p0, p1, i0, i1, i_len] + copyunicodecontent(p0, p1, i0, i1, i_len) + """, """ + [p0, p1, i0, i1, i_len] + i0s = int_lshift(i0, %(uni_itemscale)d) + i2b = int_add(p0, i0s) + i2 = int_add(i2b, %(uni_basesize)s) + i1s = int_lshift(i1, %(uni_itemscale)d) + i3b = int_add(p1, i1s) + i3 = int_add(i3b, %(uni_basesize)s) + i4 = int_lshift(i_len, %(uni_itemscale)d) + call_n(ConstClass(memcpy_fn), i3, i2, i4, descr=memcpy_descr) + """) diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -19,6 +19,7 @@ # Boxes and Consts are BoxFloats and ConstFloats. supports_singlefloats = False supports_guard_gc_type = False + supports_load_effective_address = False propagate_exception_descr = None diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py --- a/rpython/jit/backend/ppc/opassembler.py +++ b/rpython/jit/backend/ppc/opassembler.py @@ -966,72 +966,6 @@ pmc.overwrite() -class StrOpAssembler(object): - - _mixin_ = True - - def emit_copystrcontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=False) - - def emit_copyunicodecontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=True) - - def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale): - if src_ofs.is_imm(): - value = src_ofs.value << scale - if value < 32768: - self.mc.addi(dst.value, src_ptr.value, value) - else: - self.mc.load_imm(dst, value) - self.mc.add(dst.value, src_ptr.value, dst.value) - elif scale == 0: - self.mc.add(dst.value, src_ptr.value, src_ofs.value) - else: - self.mc.sldi(dst.value, src_ofs.value, scale) - self.mc.add(dst.value, src_ptr.value, dst.value) - - def _emit_copycontent(self, arglocs, is_unicode): - [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] = arglocs - - if is_unicode: - basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 2: scale = 1 - elif itemsize == 4: scale = 2 - else: raise AssertionError - else: - basesize, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - basesize -= 1 # for the extra null character - scale = 0 - - self._emit_load_for_copycontent(r.r0, src_ptr_loc, src_ofs_loc, scale) - self._emit_load_for_copycontent(r.r2, dst_ptr_loc, dst_ofs_loc, scale) - - if length_loc.is_imm(): - length = length_loc.getint() - self.mc.load_imm(r.r5, length << scale) - else: - if scale > 0: - self.mc.sldi(r.r5.value, length_loc.value, scale) - elif length_loc is not r.r5: - self.mc.mr(r.r5.value, length_loc.value) - - self.mc.mr(r.r4.value, r.r0.value) - self.mc.addi(r.r4.value, r.r4.value, basesize) - self.mc.addi(r.r3.value, r.r2.value, basesize) - - self.mc.load_imm(self.mc.RAW_CALL_REG, self.memcpy_addr) - self.mc.raw_call() - - -class UnicodeOpAssembler(object): - _mixin_ = True - # empty! - - class AllocOpAssembler(object): _mixin_ = True @@ -1336,8 +1270,7 @@ class OpAssembler(IntOpAssembler, GuardOpAssembler, MiscOpAssembler, FieldOpAssembler, - StrOpAssembler, CallOpAssembler, - UnicodeOpAssembler, ForceOpAssembler, + CallOpAssembler, ForceOpAssembler, AllocOpAssembler, FloatOpAssembler, VectorAssembler): _mixin_ = True diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py --- a/rpython/jit/backend/ppc/regalloc.py +++ b/rpython/jit/backend/ppc/regalloc.py @@ -802,18 +802,6 @@ temp_loc = r.SCRATCH2 return [base_loc, temp_loc] - def prepare_copystrcontent(self, op): - src_ptr_loc = self.ensure_reg(op.getarg(0)) - dst_ptr_loc = self.ensure_reg(op.getarg(1)) - src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) - dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) - length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - self._spill_before_call(gc_level=0) - return [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] - - prepare_copyunicodecontent = prepare_copystrcontent - prepare_same_as_i = helper.prepare_unary_op prepare_same_as_r = helper.prepare_unary_op prepare_same_as_f = helper.prepare_unary_op diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1222,78 +1222,16 @@ resloc = self.force_allocate_reg(op, [op.getarg(0)]) self.perform(op, [argloc], resloc) - def consider_copystrcontent(self, op): - self._consider_copystrcontent(op, is_unicode=False) - - def consider_copyunicodecontent(self, op): - self._consider_copystrcontent(op, is_unicode=True) - - def _consider_copystrcontent(self, op, is_unicode): - # compute the source address - args = op.getarglist() - base_loc = self.rm.make_sure_var_in_reg(args[0], args) - ofs_loc = self.rm.make_sure_var_in_reg(args[2], args) - assert args[0] is not args[1] # forbidden case of aliasing - srcaddr_box = TempVar() - forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = self.rm.force_allocate_reg(srcaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, - is_unicode=is_unicode) - # compute the destination address - base_loc = self.rm.make_sure_var_in_reg(args[1], forbidden_vars) - ofs_loc = self.rm.make_sure_var_in_reg(args[3], forbidden_vars) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempVar() - dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, forbidden_vars) - self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, - is_unicode=is_unicode) - # compute the length in bytes - length_box = args[4] - length_loc = self.loc(length_box) - if is_unicode: - forbidden_vars = [srcaddr_box, dstaddr_box] - bytes_box = TempVar() - bytes_loc = self.rm.force_allocate_reg(bytes_box, forbidden_vars) - scale = self._get_unicode_item_scale() - if not (isinstance(length_loc, ImmedLoc) or - isinstance(length_loc, RegLoc)): - self.assembler.mov(length_loc, bytes_loc) - length_loc = bytes_loc - self.assembler.load_effective_addr(length_loc, 0, scale, bytes_loc) - length_box = bytes_box - length_loc = bytes_loc - # call memcpy() - self.rm.before_call() - self.xrm.before_call() - self.assembler.simple_call_no_collect(imm(self.assembler.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) - self.rm.possibly_free_var(length_box) - self.rm.possibly_free_var(dstaddr_box) - self.rm.possibly_free_var(srcaddr_box) - - def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): - if is_unicode: - ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - scale = self._get_unicode_item_scale() - else: - ofs_items, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.translate_support_code) - assert itemsize == 1 - ofs_items -= 1 # for the extra null character - scale = 0 - self.assembler.load_effective_addr(ofsloc, ofs_items, scale, - resloc, baseloc) - - def _get_unicode_item_scale(self): - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") + def consider_load_effective_address(self, op): + p0 = op.getarg(0) + i0 = op.getarg(1) + ploc = self.make_sure_var_in_reg(p0, [i0]) + iloc = self.make_sure_var_in_reg(i0, [p0]) + res = self.rm.force_allocate_reg(op, [p0, i0]) + assert isinstance(op.getarg(2), ConstInt) + assert isinstance(op.getarg(3), ConstInt) + self.assembler.load_effective_addr(iloc, op.getarg(2).getint(), + op.getarg(3).getint(), res, ploc) def _consider_math_read_timestamp(self, op): # hint: try to move unrelated registers away from eax and edx now diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -16,6 +16,7 @@ debug = True supports_floats = True supports_singlefloats = True + supports_load_effective_address = True dont_keepalive_stuff = False # for tests with_threads = False diff --git a/rpython/jit/backend/zarch/opassembler.py b/rpython/jit/backend/zarch/opassembler.py --- a/rpython/jit/backend/zarch/opassembler.py +++ b/rpython/jit/backend/zarch/opassembler.py @@ -963,75 +963,15 @@ def _mem_offset_supported(self, value): return -2**19 <= value < 2**19 - def emit_copystrcontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=False) - - def emit_copyunicodecontent(self, op, arglocs, regalloc): - self._emit_copycontent(arglocs, is_unicode=True) - - def _emit_load_for_copycontent(self, dst, src_ptr, src_ofs, scale): - if src_ofs.is_imm(): - value = src_ofs.value << scale - if check_imm_value(value): - self.mc.AGHIK(dst, src_ptr, l.imm(value)) - else: - # it is fine to use r1 here, because it will - # only hold a value before invoking the memory copy - self.mc.load_imm(r.SCRATCH, value) - self.mc.AGRK(dst, src_ptr, r.SCRATCH) - elif scale == 0: - self.mc.AGRK(dst, src_ptr, src_ofs) - else: - self.mc.SLLG(r.SCRATCH, src_ofs, l.addr(scale)) - self.mc.AGRK(dst, src_ptr, r.SCRATCH) - - def _emit_copycontent(self, arglocs, is_unicode): - [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] = arglocs - - if is_unicode: - basesize, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - if itemsize == 2: scale = 1 - elif itemsize == 4: scale = 2 - else: raise AssertionError - else: - basesize, itemsize, _ = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - basesize -= 1 # for the extra null character - scale = 0 - - # src and src_len are tmp registers - src = src_ptr_loc - src_len = r.odd_reg(src) - dst = r.r0 - dst_len = r.r1 - self._emit_load_for_copycontent(src, src_ptr_loc, src_ofs_loc, scale) - self._emit_load_for_copycontent(dst, dst_ptr_loc, dst_ofs_loc, scale) - - if length_loc.is_imm(): - length = length_loc.getint() - self.mc.load_imm(dst_len, length << scale) - else: - if scale > 0: - self.mc.SLLG(dst_len, length_loc, l.addr(scale)) - else: - self.mc.LGR(dst_len, length_loc) - # ensure that src_len is as long as dst_len, otherwise - # padding bytes are written to dst - self.mc.LGR(src_len, dst_len) - - self.mc.AGHI(src, l.imm(basesize)) - self.mc.AGHI(dst, l.imm(basesize)) - - # s390x has memset directly as a hardware instruction!! - # 0xB8 means we might reference dst later - self.mc.MVCLE(dst, src, l.addr(0xB8)) - # NOTE this instruction can (determined by the cpu), just - # quit the movement any time, thus it is looped until all bytes - # are copied! - self.mc.BRC(c.OF, l.imm(-self.mc.MVCLE_byte_count)) + # ...copystrcontent logic was removed, but note that + # if we want to reintroduce support for that: + # s390x has memset directly as a hardware instruction!! + # 0xB8 means we might reference dst later + #self.mc.MVCLE(dst, src, l.addr(0xB8)) + # NOTE this instruction can (determined by the cpu), just + # quit the movement any time, thus it is looped until all bytes + # are copied! + #self.mc.BRC(c.OF, l.imm(-self.mc.MVCLE_byte_count)) def emit_zero_array(self, op, arglocs, regalloc): base_loc, startindex_loc, length_loc, \ diff --git a/rpython/jit/backend/zarch/regalloc.py b/rpython/jit/backend/zarch/regalloc.py --- a/rpython/jit/backend/zarch/regalloc.py +++ b/rpython/jit/backend/zarch/regalloc.py @@ -1269,29 +1269,6 @@ loc1 = self.ensure_reg(op.getarg(1)) return [loc0, loc1] - def prepare_copystrcontent(self, op): - """ this function needs five registers. - src & src_len: are allocated using ensure_even_odd_pair. - note that these are tmp registers, thus the actual variable - value is not modified. - src_len: when entering the assembler, src_ofs_loc's value is contained - in src_len register. - """ - src_ptr_loc, _ = \ - self.rm.ensure_even_odd_pair(op.getarg(0), - None, bind_first=True, - must_exist=False, load_loc_odd=False) - src_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(2)) - dst_ptr_loc = self.ensure_reg(op.getarg(1)) - dst_ofs_loc = self.ensure_reg_or_any_imm(op.getarg(3)) - length_loc = self.ensure_reg_or_any_imm(op.getarg(4)) - # no need to spill, we do not call memcpy, but we use s390x's - # hardware instruction to copy memory - return [src_ptr_loc, dst_ptr_loc, - src_ofs_loc, dst_ofs_loc, length_loc] - - prepare_copyunicodecontent = prepare_copystrcontent - def prepare_label(self, op): descr = op.getdescr() assert isinstance(descr, TargetToken) diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -441,6 +441,7 @@ rop.GC_STORE, rop.GC_STORE_INDEXED, rop.LOAD_FROM_GC_TABLE, + rop.LOAD_EFFECTIVE_ADDRESS, ): # list of opcodes never executed by pyjitpl continue if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST: diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -629,6 +629,20 @@ # and then emit the operation return self.emit(op) + def optimize_GC_LOAD_I(self, op): + # seeing a 'gc_load*' forces all the lazy sets that are still + # pending, as an approximation. We could try to be really clever + # and only force some of them, but we don't have any descr here. + self.force_all_lazy_sets() + self.make_nonnull(op.getarg(0)) + return self.emit(op) + optimize_GC_LOAD_R = optimize_GC_LOAD_I + optimize_GC_LOAD_F = optimize_GC_LOAD_I + + optimize_GC_LOAD_INDEXED_I = optimize_GC_LOAD_I + optimize_GC_LOAD_INDEXED_R = optimize_GC_LOAD_I + optimize_GC_LOAD_INDEXED_F = optimize_GC_LOAD_I + def optimize_QUASIIMMUT_FIELD(self, op): # Pattern: QUASIIMMUT_FIELD(s, descr=QuasiImmutDescr) # x = GETFIELD_GC(s, descr='inst_x') # pure diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -9495,3 +9495,42 @@ jump(i3, i2, i3) """ self.optimize_loop(ops, expected) + + def test_issue3014(self): + # 'gc_load_indexed' must force 'setarrayitem_gc' + ops = """ + [i183] + p0 = new_array(5, descr=arraydescr) + setarrayitem_gc(p0, 0, i183, descr=arraydescr) + i235 = gc_load_indexed_i(p0, 0, 1, 16, 2) + escape_i(i235) + jump(i183) + """ + self.optimize_loop(ops, ops) + + def test_issue3014_2(self): + # same rules for gc_store_indexed versus getarrayitem_gc, + # and 'gc_store_indexed' invalidates the value for 'getarrayitem_gc' + # (in this direction it seems to work already) + ops = """ + [p0, i183] + i234 = getarrayitem_gc_i(p0, 0, descr=arraydescr) + gc_store_indexed(p0, 0, i183, 1, 16, 2) + i235 = getarrayitem_gc_i(p0, 0, descr=arraydescr) + escape_i(i234) + escape_i(i235) + jump(p0, i183) + """ + self.optimize_loop(ops, ops) + + def test_issue3014_3(self): + # 'gc_load' must force 'setfield_gc' + ops = """ + [i183] + p0 = new(descr=ssize) + setfield_gc(p0, i183, descr=adescr) + i235 = gc_load_i(p0, 8, 2) + escape_i(i235) + jump(i183) + """ + self.optimize_loop(ops, ops) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -1055,6 +1055,8 @@ 'UNICODEGETITEM/2/i', # 'LOAD_FROM_GC_TABLE/1/r', # only emitted by rewrite.py + 'LOAD_EFFECTIVE_ADDRESS/4/i', # only emitted by rewrite.py, only if + # cpu.supports_load_effective_address. [v_gcptr,v_index,c_baseofs,c_shift] # '_ALWAYS_PURE_LAST', # ----- end of always_pure operations ----- diff --git a/rpython/rtyper/test/test_rfloat.py b/rpython/rtyper/test/test_rfloat.py --- a/rpython/rtyper/test/test_rfloat.py +++ b/rpython/rtyper/test/test_rfloat.py @@ -253,3 +253,10 @@ return compute_hash(f) res = self.interpret(fn, [1.5]) assert res == compute_hash(1.5) + + def test_float_constant_inf(self): + from rpython.rlib.rfloat import INFINITY + def fn(): + return INFINITY # float('inf') is not supported by RPython so far + res = self.interpret(fn, []) + assert res == float('inf') From pypy.commits at gmail.com Tue Jun 11 07:21:34 2019 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 11 Jun 2019 04:21:34 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps-py3.6: 3.6 version of json-decoder-maps Message-ID: <5cff8ebe.1c69fb81.8ea19.cd54@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps-py3.6 Changeset: r96790:5bea5a7b9381 Date: 2019-06-11 13:20 +0200 http://bitbucket.org/pypy/pypy/changeset/5bea5a7b9381/ Log: 3.6 version of json-decoder-maps diff too long, truncating to 2000 out of 2070 lines diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -49,3 +49,9 @@ Instead, replace it in ``rewrite.py`` with a direct call to ``memcpy()`` and new basic operation, ``load_effective_address``, which the backend can even decide not to implement. + +.. branch: json-decoder-maps + +Much faster and more memory-efficient JSON decoding. The resulting +dictionaries that come out of the JSON decoder have faster lookups too. + diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -1,11 +1,13 @@ import sys from rpython.rlib.rstring import StringBuilder -from rpython.rlib.objectmodel import specialize, always_inline, r_dict -from rpython.rlib import rfloat, rutf8 +from rpython.rlib.objectmodel import specialize, always_inline +from rpython.rlib import rfloat, runicode, jit, objectmodel, rutf8 from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import oefmt, OperationError from rpython.rlib.rarithmetic import r_uint from pypy.interpreter import unicodehelper +from pypy.interpreter.baseobjspace import W_Root +from pypy.module._pypyjson import simd OVF_DIGITS = len(str(sys.maxint)) @@ -15,50 +17,97 @@ # precomputing negative powers of 10 is MUCH faster than using e.g. math.pow # at runtime NEG_POW_10 = [10.0**-i for i in range(16)] +del i + def neg_pow_10(x, exp): if exp >= len(NEG_POW_10): return 0.0 return x * NEG_POW_10[exp] -def slice_eq(a, b): - (ll_chars1, start1, length1, _) = a - (ll_chars2, start2, length2, _) = b - if length1 != length2: - return False - j = start2 - for i in range(start1, start1 + length1): - if ll_chars1[i] != ll_chars2[j]: - return False - j += 1 - return True -def slice_hash(a): - (ll_chars, start, length, h) = a - return h +class IntCache(object): + """ A cache for wrapped ints between START and END """ + + START = -10 + END = 256 + + def __init__(self, space): + self.space = space + self.cache = [self.space.newint(i) + for i in range(self.START, self.END)] + + def newint(self, intval): + if self.START <= intval < self.END: + return self.cache[intval - self.START] + return self.space.newint(intval) class DecoderError(Exception): def __init__(self, msg, pos): self.msg = msg self.pos = pos -TYPE_UNKNOWN = 0 -TYPE_STRING = 1 -class JSONDecoder(object): +class JSONDecoder(W_Root): + + LRU_SIZE = 16 + LRU_MASK = LRU_SIZE - 1 + + DEFAULT_SIZE_SCRATCH = 20 + + MIN_SIZE_FOR_STRING_CACHE = 1024 * 1024 + + # evaluate the string cache for 200 strings, before looking at the hit rate + # and deciding whether to keep doing it + STRING_CACHE_EVALUATION_SIZE = 200 + + # keep using the string cache if at least 25% of all decoded strings are a + # hit in the cache + STRING_CACHE_USEFULNESS_FACTOR = 4 + + def __init__(self, space, s): self.space = space + self.w_empty_string = space.newutf8("", 0) + self.s = s + # we put our string in a raw buffer so: # 1) we automatically get the '\0' sentinel at the end of the string, # which means that we never have to check for the "end of string" # 2) we can pass the buffer directly to strtod - self.ll_chars = rffi.str2charp(s) + self.ll_chars, self.flag = rffi.get_nonmovingbuffer_final_null(self.s) self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') self.pos = 0 - self.cache = r_dict(slice_eq, slice_hash, simple_hash_eq=True) + self.intcache = space.fromcache(IntCache) + + # two caches, one for keys, one for general strings. they both have the + # form {hash-as-int: CacheEntry} and they don't deal with + # collisions at all. For every hash there is simply one string stored. + self.cache = {} + self.cache_wrapped = {} + + self.lru_cache = [0] * self.LRU_SIZE + self.lru_index = 0 + + self.startmap = self.space.fromcache(Terminator) + + # keep a list of objects that are created with maps that aren't clearly + # useful. If they turn out to be useful in the end we are good, + # otherwise convert them to dicts (see .close()) + self.unclear_objects = [] + + # this is a freelist of lists that store the decoded value of an + # object, before they get copied into the eventual dict + self.scratch = [[None] * self.DEFAULT_SIZE_SCRATCH] + def close(self): - rffi.free_charp(self.ll_chars) + rffi.free_nonmovingbuffer(self.s, self.ll_chars, self.flag) lltype.free(self.end_ptr, flavor='raw') + # clean up objects that are instances of now blocked maps + for w_obj in self.unclear_objects: + jsonmap = self._get_jsonmap_from_dict(w_obj) + if jsonmap.is_state_blocked(): + self._devolve_jsonmap_dict(w_obj) def getslice(self, start, end): assert start >= 0 @@ -66,19 +115,22 @@ return self.s[start:end] def skip_whitespace(self, i): + ll_chars = self.ll_chars while True: - ch = self.ll_chars[i] + ch = ll_chars[i] if is_whitespace(ch): - i+=1 + i += 1 else: break return i - def decode_any(self, i): + def decode_any(self, i, contextmap=None): + """ Decode an object at position i. Optionally pass a contextmap, if + the value is decoded as the value of a dict. """ i = self.skip_whitespace(i) ch = self.ll_chars[i] if ch == '"': - return self.decode_string(i+1) + return self.decode_string(i+1, contextmap) elif ch == '[': return self.decode_array(i+1) elif ch == '{': @@ -100,7 +152,11 @@ elif ch.isdigit(): return self.decode_numeric(i) else: - raise DecoderError("Unexpected '%s' at" % ch, i) + raise DecoderError("Unexpected '%s'" % ch, i) + + + def _raise(self, msg, pos): + raise DecoderError(msg, pos) def decode_null(self, i): if (self.ll_chars[i] == 'u' and @@ -108,7 +164,7 @@ self.ll_chars[i+2] == 'l'): self.pos = i+3 return self.space.w_None - raise DecoderError("Error when decoding null at", i) + raise DecoderError("Error when decoding null", i) def decode_true(self, i): if (self.ll_chars[i] == 'r' and @@ -116,7 +172,7 @@ self.ll_chars[i+2] == 'e'): self.pos = i+3 return self.space.w_True - raise DecoderError("Error when decoding true at", i) + raise DecoderError("Error when decoding true", i) def decode_false(self, i): if (self.ll_chars[i] == 'a' and @@ -125,7 +181,7 @@ self.ll_chars[i+3] == 'e'): self.pos = i+4 return self.space.w_False - raise DecoderError("Error when decoding false at", i) + raise DecoderError("Error when decoding false", i) def decode_infinity(self, i, sign=1): if (self.ll_chars[i] == 'n' and @@ -137,14 +193,14 @@ self.ll_chars[i+6] == 'y'): self.pos = i+7 return self.space.newfloat(rfloat.INFINITY * sign) - raise DecoderError("Error when decoding Infinity at", i) + raise DecoderError("Error when decoding Infinity", i) def decode_nan(self, i): if (self.ll_chars[i] == 'a' and self.ll_chars[i+1] == 'N'): self.pos = i+2 return self.space.newfloat(rfloat.NAN) - raise DecoderError("Error when decoding NaN at", i) + raise DecoderError("Error when decoding NaN", i) def decode_numeric(self, i): start = i @@ -154,7 +210,7 @@ ch = self.ll_chars[i] if ch == '.': if not self.ll_chars[i+1].isdigit(): - raise DecoderError("Expected digit at", i+1) + raise DecoderError("Expected digit", i+1) return self.decode_float(start) elif ch == 'e' or ch == 'E': return self.decode_float(start) @@ -162,7 +218,7 @@ return self.decode_int_slow(start) self.pos = i - return self.space.newint(intval) + return self.intcache.newint(intval) def decode_float(self, i): from rpython.rlib import rdtoa @@ -208,13 +264,27 @@ break count = i - start if count == 0: - raise DecoderError("Expected digit at", i) + raise DecoderError("Expected digit", i) # if the number has more digits than OVF_DIGITS, it might have # overflowed ovf_maybe = (count >= OVF_DIGITS) return i, ovf_maybe, sign * intval + def _raise_control_char_in_string(self, ch, startindex, currindex): + if ch == '\0': + self._raise("Unterminated string starting at", + startindex - 1) + else: + self._raise("Invalid control character at", currindex-1) + + def _raise_object_error(self, ch, start, i): + if ch == '\0': + self._raise("Unterminated object starting at", start) + else: + self._raise("Unexpected '%s' when decoding object (char %d)" % ch, i) + def decode_array(self, i): + """ Decode a list. i must be after the opening '[' """ w_list = self.space.newlist([]) start = i i = self.skip_whitespace(start) @@ -248,63 +318,120 @@ self.pos = i+1 return self.space.newdict() - d = self._create_empty_dict() + if self.scratch: + values_w = self.scratch.pop() + else: + values_w = [None] * self.DEFAULT_SIZE_SCRATCH + nextindex = 0 + currmap = self.startmap while True: # parse a key: value - w_name = self.decode_key(i) + currmap = self.decode_key_map(i, currmap) i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] if ch != ':': raise DecoderError("No ':' found at", i) i += 1 - i = self.skip_whitespace(i) - # - w_value = self.decode_any(i) - d[w_name] = w_value + + w_value = self.decode_any(i, currmap) + + if nextindex == len(values_w): # full + values_w = values_w + [None] * len(values_w) # double + values_w[nextindex] = w_value + nextindex += 1 i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] i += 1 if ch == '}': self.pos = i - return self._create_dict(d) + self.scratch.append(values_w) # can reuse next time + if currmap.is_state_blocked(): + dict_w = self._switch_to_dict(currmap, values_w, nextindex) + return self._create_dict(dict_w) + values_w = values_w[:nextindex] + w_res = self._create_dict_map(values_w, currmap) + if not currmap.is_state_useful(): + self.unclear_objects.append(w_res) + return w_res elif ch == ',': - pass - elif ch == '\0': - raise DecoderError("Unterminated object starting at", start) + i = self.skip_whitespace(i) + if currmap.is_state_blocked(): + self.scratch.append(values_w) # can reuse next time + dict_w = self._switch_to_dict(currmap, values_w, nextindex) + return self.decode_object_dict(i, start, dict_w) else: - raise DecoderError("Unexpected '%s' when decoding object" % ch, - i-1) + self._raise_object_error(ch, start, i - 1) - def decode_string(self, i): - start = i - bits = 0 + def _create_dict_map(self, values_w, jsonmap): + from pypy.objspace.std.jsondict import from_values_and_jsonmap + return from_values_and_jsonmap(self.space, values_w, jsonmap) + + def _devolve_jsonmap_dict(self, w_dict): + from pypy.objspace.std.jsondict import devolve_jsonmap_dict + devolve_jsonmap_dict(w_dict) + + def _get_jsonmap_from_dict(self, w_dict): + from pypy.objspace.std.jsondict import get_jsonmap_from_dict + return get_jsonmap_from_dict(w_dict) + + def _switch_to_dict(self, currmap, values_w, nextindex): + dict_w = self._create_empty_dict() + index = nextindex - 1 + while isinstance(currmap, JSONMap): + dict_w[currmap.w_key] = values_w[index] + index -= 1 + currmap = currmap.prev + assert len(dict_w) == nextindex + return dict_w + + def decode_object_dict(self, i, start, dict_w): while True: - # this loop is a fast path for strings which do not contain escape - # characters + # parse a key: value + w_key = self.decode_key_string(i) + i = self.skip_whitespace(self.pos) + ch = self.ll_chars[i] + if ch != ':': + self._raise("No ':' found at", i) + i += 1 + + w_value = self.decode_any(i) + dict_w[w_key] = w_value + i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] i += 1 - bits |= ord(ch) - if ch == '"': + if ch == '}': self.pos = i - return self._create_string(start, i - 1, bits) - elif ch == '\\' or ch < '\x20': - self.pos = i-1 - return self.decode_string_escaped(start) + return self._create_dict(dict_w) + elif ch == ',': + i = self.skip_whitespace(i) + else: + self._raise_object_error(ch, start, i - 1) - def _create_string(self, start, end, bits): - if bits & 0x80: - # the 8th bit is set, it's an utf8 string - content_utf8 = self.getslice(start, end) + def decode_string_uncached(self, i): + start = i + ll_chars = self.ll_chars + nonascii, i = simd.find_end_of_string_no_hash(ll_chars, i, len(self.s)) + ch = ll_chars[i] + if ch == '\\': + self.pos = i + return self.decode_string_escaped(start, nonascii) + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + else: + assert ch == '"' + + self.pos = i + 1 + return self._create_string_wrapped(start, i, nonascii) + + def _create_string_wrapped(self, start, end, nonascii): + content = self.getslice(start, end) + if nonascii: + # contains non-ascii chars, we need to check that it's valid utf-8 lgt = unicodehelper.check_utf8_or_raise(self.space, - content_utf8) - return self.space.newutf8(content_utf8, lgt) + content) else: - # ascii only, fast path (ascii is a strict subset of - # latin1, and we already checked that all the chars are < - # 128) lgt = end - start - assert lgt >= 0 - return self.space.newutf8(self.getslice(start, end), lgt) + return self.space.newutf8(content, lgt) def _create_dict(self, d): from pypy.objspace.std.dictmultiobject import from_unicode_key_dict @@ -314,8 +441,7 @@ from pypy.objspace.std.dictmultiobject import create_empty_unicode_key_dict return create_empty_unicode_key_dict(self.space) - - def decode_string_escaped(self, start): + def decode_string_escaped(self, start, nonascii): i = self.pos builder = StringBuilder((i - start) * 2) # just an estimate assert start >= 0 @@ -326,25 +452,21 @@ i += 1 if ch == '"': content_utf8 = builder.build() - lgt = unicodehelper.check_utf8_or_raise(self.space, + length = unicodehelper.check_utf8_or_raise(self.space, content_utf8) self.pos = i - return self.space.newutf8(content_utf8, lgt) + return self.space.newutf8(content_utf8, length) elif ch == '\\': - i = self.decode_escape_sequence(i, builder) + i = self.decode_escape_sequence_to_utf8(i, builder) elif ch < '\x20': - if ch == '\0': - raise DecoderError("Unterminated string starting at", - start - 1) - else: - raise DecoderError("Invalid control character at", i-1) + self._raise_control_char_in_string(ch, start, i) else: builder.append(ch) - def decode_escape_sequence(self, i, builder): + def decode_escape_sequence_to_utf8(self, i, stringbuilder): ch = self.ll_chars[i] i += 1 - put = builder.append + put = stringbuilder.append if ch == '\\': put('\\') elif ch == '"': put('"' ) elif ch == '/': put('/' ) @@ -354,22 +476,37 @@ elif ch == 'r': put('\r') elif ch == 't': put('\t') elif ch == 'u': - return self.decode_escape_sequence_unicode(i, builder) + # may be a surrogate pair + return self.decode_escape_sequence_unicode(i, stringbuilder) else: raise DecoderError("Invalid \\escape: %s" % ch, i-1) return i + def _get_int_val_from_hex4(self, i): + ll_chars = self.ll_chars + res = 0 + for i in range(i, i + 4): + ch = ord(ll_chars[i]) + if ord('a') <= ch <= ord('f'): + digit = ch - ord('a') + 10 + elif ord('A') <= ch <= ord('F'): + digit = ch - ord('A') + 10 + elif ord('0') <= ch <= ord('9'): + digit = ch - ord('0') + else: + raise ValueError + res = (res << 4) + digit + return res + def decode_escape_sequence_unicode(self, i, builder): # at this point we are just after the 'u' of the \u1234 sequence. start = i i += 4 - hexdigits = self.getslice(start, i) try: - val = int(hexdigits, 16) + val = self._get_int_val_from_hex4(start) if (0xd800 <= val <= 0xdbff and self.ll_chars[i] == '\\' and self.ll_chars[i+1] == 'u'): - hexdigits = self.getslice(i+2, i+6) - lowsurr = int(hexdigits, 16) + lowsurr = self._get_int_val_from_hex4(i + 2) if 0xdc00 <= lowsurr <= 0xdfff: # decode surrogate pair val = 0x10000 + (((val - 0xd800) << 10) | @@ -384,45 +521,585 @@ builder.append(utf8_ch) return i - def decode_key(self, i): - """ returns a wrapped unicode """ - from rpython.rlib.rarithmetic import intmask - i = self.skip_whitespace(i) + def decode_string(self, i, contextmap=None): + """ Decode a string at position i (which is right after the opening "). + Optionally pass a contextmap, if the value is decoded as the value of a + dict. """ ll_chars = self.ll_chars + start = i + ch = ll_chars[i] + if ch == '"': + self.pos = i + 1 + return self.w_empty_string # surprisingly common + + cache = True + if contextmap is not None: + contextmap.decoded_strings += 1 + if not contextmap.should_cache_strings(): + cache = False + if len(self.s) < self.MIN_SIZE_FOR_STRING_CACHE: + cache = False + + if not cache: + return self.decode_string_uncached(i) + + strhash, nonascii, i = simd.find_end_of_string(ll_chars, i, len(self.s)) + ch = ll_chars[i] + if ch == '\\': + self.pos = i + return self.decode_string_escaped(start, nonascii) + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + else: + assert ch == '"' + + self.pos = i + 1 + + length = i - start + strhash ^= length + + # check cache first: + try: + entry = self.cache_wrapped[strhash] + except KeyError: + w_res = self._create_string_wrapped(start, i, nonascii) + # only add *some* strings to the cache, because keeping them all is + # way too expensive + if ((contextmap is not None and + contextmap.decoded_strings < self.STRING_CACHE_EVALUATION_SIZE) or + strhash in self.lru_cache): + entry = CacheEntry( + self.getslice(start, start + length), w_res) + self.cache_wrapped[strhash] = entry + else: + self.lru_cache[self.lru_index] = strhash + self.lru_index = (self.lru_index + 1) & self.LRU_MASK + return w_res + if not entry.compare(ll_chars, start, length): + # collision! hopefully rare + return self._create_string_wrapped(start, i, nonascii) + if contextmap is not None: + contextmap.cache_hits += 1 + return entry.w_uni + + def decode_key_map(self, i, currmap): + """ Given the current map currmap of an object, decode the next key at + position i. This returns the new map of the object. """ + newmap = self._decode_key_map(i, currmap) + currmap.observe_transition(newmap, self.startmap) + return newmap + + def _decode_key_map(self, i, currmap): + ll_chars = self.ll_chars + # first try to see whether we happen to find currmap.single_nextmap + nextmap = currmap.fast_path_key_parse(self, i) + if nextmap is not None: + return nextmap + + start = i ch = ll_chars[i] if ch != '"': raise DecoderError("Key name must be string at char", i) i += 1 + w_key = self._decode_key_string(i) + return currmap.get_next(w_key, self.s, start, self.pos, self.startmap) + def _decode_key_string(self, i): + """ decode key at position i as a string. Key strings are always + cached, since they repeat a lot. """ + ll_chars = self.ll_chars start = i - bits = 0 - strhash = ord(ll_chars[i]) << 7 - while True: - ch = ll_chars[i] + + strhash, nonascii, i = simd.find_end_of_string(ll_chars, i, len(self.s)) + + ch = ll_chars[i] + if ch == '\\': + self.pos = i + w_key = self.decode_string_escaped(start, nonascii) + return w_key + if ch < '\x20': + self._raise_control_char_in_string(ch, start, i) + length = i - start + strhash ^= length + self.pos = i + 1 + # check cache first: + try: + entry = self.cache[strhash] + except KeyError: + w_res = self._create_string_wrapped(start, i, nonascii) + entry = CacheEntry( + self.getslice(start, start + length), w_res) + self.cache[strhash] = entry + return w_res + if not entry.compare(ll_chars, start, length): + # collision! hopefully rare + w_res = self._create_string_wrapped(start, i, nonascii) + else: + w_res = entry.w_uni + return w_res + + def decode_key_string(self, i): + ll_chars = self.ll_chars + ch = ll_chars[i] + if ch != '"': + self._raise("Key name must be string at char %d", i) + i += 1 + return self._decode_key_string(i) + +class CacheEntry(object): + """ A cache entry, bundling the encoded version of a string, and its wrapped + decoded variant. """ + def __init__(self, repr, w_uni): + # repr is the escaped string + self.repr = repr + # uni is the wrapped decoded string + self.w_uni = w_uni + + def compare(self, ll_chars, start, length): + """ Check whether self.repr occurs at ll_chars[start:start+length] """ + if length != len(self.repr): + return False + index = start + for c in self.repr: + if not ll_chars[index] == c: + return False + index += 1 + return True + + +class MapBase(object): + """ A map implementation to speed up parsing of json dicts, and to + represent the resulting dicts more compactly and make access faster. """ + + # the basic problem we are trying to solve is the following: dicts in + # json can either be used as objects, or as dictionaries with arbitrary + # string keys. We want to use maps for the former, but not for the + # latter. But we don't know in advance which kind of dict is which. + + # Therefore we create "preliminary" maps where we aren't quite sure yet + # whether they are really useful maps or not. If we see them used often + # enough, we promote them to "useful" maps, which we will actually + # instantiate objects with. + + # If we determine that a map is not used often enough, we can turn it + # into a "blocked" map, which is a point in the map tree where we will + # switch to regular dicts, when we reach that part of the tree. + + # One added complication: We want to keep the number of preliminary maps + # bounded to prevent generating tons of useless maps. but also not too + # small, to support having a json file that contains many uniform objects + # with tons of keys. That's where the idea of "fringe" maps comes into + # play. They are maps that sit between known useful nodes and preliminary + # nodes in the map transition tree. We bound only the number of fringe + # nodes we are considering (to MAX_FRINGE), but not the number of + # preliminary maps. When we have too many fringe maps, we remove the least + # commonly instantiated fringe map and mark it as blocked. + + # allowed graph edges or nodes in all_next: + # USEFUL ------- + # / \ \ + # v v v + # FRINGE USEFUL BLOCKED + # | + # v + # PRELIMINARY + # | + # v + # PRELIMINARY + + # state transitions: + # PRELIMINARY + # / | \ + # | v v + # | FRINGE -> USEFUL + # | | + # \ | + # v v + # BLOCKED + + # the single_nextmap edge can only be these graph edges: + # USEFUL + # | + # v + # USEFUL + # + # FRINGE + # | + # v + # PRELIMINARY + # | + # v + # PRELIMINARY + + USEFUL = 'u' + PRELIMINARY = 'p' + FRINGE = 'f' # buffer between PRELIMINARY and USEFUL + BLOCKED = 'b' + + # tunable parameters + MAX_FRINGE = 40 + USEFUL_THRESHOLD = 5 + + def __init__(self, space): + self.space = space + + # a single transition is stored in .single_nextmap + self.single_nextmap = None + + # all_next is only initialized after seeing the *second* transition + # but then it also contains .single_nextmap + self.all_next = None # later dict {key: nextmap} + + # keep some statistics about every map: how often it was instantiated + # and how many non-blocked leaves the map transition tree has, starting + # from self + self.instantiation_count = 0 + self.number_of_leaves = 1 + + def _check_invariants(self): + if self.all_next: + for next in self.all_next.itervalues(): + next._check_invariants() + elif self.single_nextmap: + self.single_nextmap._check_invariants() + + def get_next(self, w_key, string, start, stop, terminator): + from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq + if isinstance(self, JSONMap): + assert not self.state == MapBase.BLOCKED + single_nextmap = self.single_nextmap + if (single_nextmap is not None and + single_nextmap.w_key.eq_w(w_key)): + return single_nextmap + + assert stop >= 0 + assert start >= 0 + + if single_nextmap is None: + # first transition ever seen, don't initialize all_next + next = self._make_next_map(w_key, string[start:stop]) + self.single_nextmap = next + else: + if self.all_next is None: + self.all_next = objectmodel.r_dict(unicode_eq, unicode_hash, + force_non_null=True, simple_hash_eq=True) + self.all_next[single_nextmap.w_key] = single_nextmap + else: + next = self.all_next.get(w_key, None) + if next is not None: + return next + next = self._make_next_map(w_key, string[start:stop]) + self.all_next[w_key] = next + + # one new leaf has been created + self.change_number_of_leaves(1) + + terminator.register_potential_fringe(next) + return next + + def change_number_of_leaves(self, difference): + """ add difference to .number_of_leaves of self and its parents """ + if not difference: + return + parent = self + while isinstance(parent, JSONMap): + parent.number_of_leaves += difference + parent = parent.prev + parent.number_of_leaves += difference # terminator + + def fast_path_key_parse(self, decoder, position): + """ Fast path when parsing the next key: We speculate that we will + always see a commonly seen next key, and use strcmp (implemented in + key_repr_cmp) to check whether that is the case. """ + single_nextmap = self.single_nextmap + if single_nextmap: + ll_chars = decoder.ll_chars + assert isinstance(single_nextmap, JSONMap) + if single_nextmap.key_repr_cmp(ll_chars, position): + decoder.pos = position + len(single_nextmap.key_repr) + return single_nextmap + + def observe_transition(self, newmap, terminator): + """ observe a transition from self to newmap. + This does a few things, including updating the self size estimate with + the knowledge that one object transitioned from self to newmap. + also it potentially decides that self should move to state USEFUL.""" + newmap.instantiation_count += 1 + if isinstance(self, JSONMap) and self.state == MapBase.FRINGE: + if self.is_useful(): + self.mark_useful(terminator) + + def _make_next_map(self, w_key, key_repr): + return JSONMap(self.space, self, w_key, key_repr) + + def _all_dot(self, output): + identity = objectmodel.compute_unique_id(self) + output.append('%s [shape=box%s];' % (identity, self._get_dot_text())) + if self.all_next: + for w_key, value in self.all_next.items(): + assert isinstance(value, JSONMap) + if value is self.single_nextmap: + color = ", color=blue" + else: + color = "" + output.append('%s -> %s [label="%s"%s];' % ( + identity, objectmodel.compute_unique_id(value), value.w_key._utf8, color)) + value._all_dot(output) + elif self.single_nextmap is not None: + value = self.single_nextmap + output.append('%s -> %s [label="%s", color=blue];' % ( + identity, objectmodel.compute_unique_id(value), value.w_key._utf8)) + value._all_dot(output) + + + def _get_dot_text(self): + return ", label=base" + + def view(self): + from dotviewer import graphclient + import pytest + r = ["digraph G {"] + self._all_dot(r) + r.append("}") + p = pytest.ensuretemp("jsonmap").join("temp.dot") + p.write("\n".join(r)) + graphclient.display_dot_file(str(p)) + + +class Terminator(MapBase): + """ The root node of the map transition tree. """ + def __init__(self, space): + MapBase.__init__(self, space) + # a set of all map nodes that are currently in the FRINGE state + self.current_fringe = {} + + def register_potential_fringe(self, prelim): + """ add prelim to the fringe, if its prev is either a Terminator or + useful. """ + prev = prelim.prev + if (isinstance(prev, Terminator) or + isinstance(prev, JSONMap) and prev.state == MapBase.USEFUL): + assert prelim.state == MapBase.PRELIMINARY + prelim.state = MapBase.FRINGE + + if len(self.current_fringe) > MapBase.MAX_FRINGE: + self.cleanup_fringe() + self.current_fringe[prelim] = None + + def remove_from_fringe(self, former_fringe): + """ Remove former_fringe from self.current_fringe. """ + assert former_fringe.state in (MapBase.USEFUL, MapBase.BLOCKED) + del self.current_fringe[former_fringe] + + def cleanup_fringe(self): + """ remove the least-instantiated fringe map and block it.""" + min_fringe = None + min_avg = 1e200 + for f in self.current_fringe: + assert f.state == MapBase.FRINGE + avg = f.average_instantiation() + if avg < min_avg: + min_avg = avg + min_fringe = f + assert min_fringe + min_fringe.mark_blocked(self) + + def _check_invariants(self): + for fringe in self.current_fringe: + assert fringe.state == MapBase.FRINGE + +class JSONMap(MapBase): + """ A map implementation to speed up parsing """ + + def __init__(self, space, prev, w_key, key_repr): + MapBase.__init__(self, space) + + self.prev = prev + self.w_key = w_key + self.key_repr = key_repr + + self.state = MapBase.PRELIMINARY + + # key decoding stats + self.decoded_strings = 0 + self.cache_hits = 0 + + # for jsondict support + self.key_to_index = None + self.keys_in_order = None + self.strategy_instance = None + + def __repr__(self): + return "" % ( + self.key_repr, self.instantiation_count, self.number_of_leaves, self.prev) + + def _get_terminator(self): # only for _check_invariants + while isinstance(self, JSONMap): + self = self.prev + assert isinstance(self, Terminator) + return self + + def _check_invariants(self): + assert self.state in ( + MapBase.USEFUL, + MapBase.PRELIMINARY, + MapBase.FRINGE, + MapBase.BLOCKED, + ) + + prev = self.prev + if isinstance(prev, JSONMap): + prevstate = prev.state + else: + prevstate = MapBase.USEFUL + + if prevstate == MapBase.USEFUL: + assert self.state != MapBase.PRELIMINARY + elif prevstate == MapBase.PRELIMINARY: + assert self.state == MapBase.PRELIMINARY + elif prevstate == MapBase.FRINGE: + assert self.state == MapBase.PRELIMINARY + else: + # if prevstate is BLOCKED, we shouldn't have recursed here! + assert False, "should be unreachable" + + if self.state == MapBase.BLOCKED: + assert self.single_nextmap is None + assert self.all_next is None + elif self.state == MapBase.FRINGE: + assert self in self._get_terminator().current_fringe + + MapBase._check_invariants(self) + + def mark_useful(self, terminator): + """ mark self as useful, and also the most commonly instantiated + children, recursively """ + was_fringe = self.state == MapBase.FRINGE + assert self.state in (MapBase.FRINGE, MapBase.PRELIMINARY) + self.state = MapBase.USEFUL + if was_fringe: + terminator.remove_from_fringe(self) + # find the most commonly instantiated child, store it into + # single_nextmap and mark it useful, recursively + maxchild = self.single_nextmap + if self.all_next is not None: + for child in self.all_next.itervalues(): + if child.instantiation_count > maxchild.instantiation_count: + maxchild = child + if maxchild is not None: + maxchild.mark_useful(terminator) + if self.all_next: + for child in self.all_next.itervalues(): + if child is not maxchild: + terminator.register_potential_fringe(child) + self.single_nextmap = maxchild + + def mark_blocked(self, terminator): + """ mark self and recursively all its children as blocked.""" + was_fringe = self.state == MapBase.FRINGE + self.state = MapBase.BLOCKED + if was_fringe: + terminator.remove_from_fringe(self) + if self.all_next: + for next in self.all_next.itervalues(): + next.mark_blocked(terminator) + elif self.single_nextmap: + self.single_nextmap.mark_blocked(terminator) + self.single_nextmap = None + self.all_next = None + self.change_number_of_leaves(-self.number_of_leaves + 1) + + def is_state_blocked(self): + return self.state == MapBase.BLOCKED + + def is_state_useful(self): + return self.state == MapBase.USEFUL + + def average_instantiation(self): + """ the number of instantiations, divided by the number of leaves. We + want to favor nodes that have either a high instantiation count, or few + leaves below it. """ + return self.instantiation_count / float(self.number_of_leaves) + + def is_useful(self): + return self.average_instantiation() > self.USEFUL_THRESHOLD + + def should_cache_strings(self): + """ return whether strings parsed in the context of this map should be + cached. """ + # we should cache if either we've seen few strings so far (less than + # STRING_CACHE_EVALUATION_SIZE), or if we've seen many, and the cache + # hit rate has been high enough + return not (self.decoded_strings > JSONDecoder.STRING_CACHE_EVALUATION_SIZE and + self.cache_hits * JSONDecoder.STRING_CACHE_USEFULNESS_FACTOR < self.decoded_strings) + + def key_repr_cmp(self, ll_chars, i): + for j, c in enumerate(self.key_repr): + if ll_chars[i] != c: + return False i += 1 - if ch == '"': - break - elif ch == '\\' or ch < '\x20': - self.pos = i-1 - return self.decode_string_escaped(start) - strhash = intmask((1000003 * strhash) ^ ord(ll_chars[i])) - bits |= ord(ch) - length = i - start - 1 - if length == 0: - strhash = -1 + return True + + # _____________________________________________________ + # methods for JsonDictStrategy + + @jit.elidable + def get_index(self, w_key): + from pypy.objspace.std.unicodeobject import W_UnicodeObject + assert isinstance(w_key, W_UnicodeObject) + return self.get_key_to_index().get(w_key, -1) + + def get_key_to_index(self): + from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq + key_to_index = self.key_to_index + if key_to_index is None: + key_to_index = self.key_to_index = objectmodel.r_dict(unicode_eq, unicode_hash, + force_non_null=True, simple_hash_eq=True) + # compute depth + curr = self + depth = 0 + while True: + depth += 1 + curr = curr.prev + if not isinstance(curr, JSONMap): + break + + curr = self + while depth: + depth -= 1 + key_to_index[curr.w_key] = depth + curr = curr.prev + if not isinstance(curr, JSONMap): + break + return key_to_index + + def get_keys_in_order(self): + keys_in_order = self.keys_in_order + if keys_in_order is None: + key_to_index = self.get_key_to_index() + keys_in_order = self.keys_in_order = [None] * len(key_to_index) + for w_key, index in key_to_index.iteritems(): + keys_in_order[index] = w_key + return keys_in_order + + # _____________________________________________________ + + def _get_dot_text(self): + if self.all_next is None: + l = int(self.single_nextmap is not None) else: - strhash ^= length - strhash = intmask(strhash) - self.pos = i - # check cache first: - key = (ll_chars, start, length, strhash) - try: - return self.cache[key] - except KeyError: - pass - res = self._create_string(start, i - 1, bits) - self.cache[key] = res + l = len(self.all_next) + extra = "" + if self.decoded_strings: + extra = "\\n%s/%s (%s%%)" % (self.cache_hits, self.decoded_strings, self.cache_hits/float(self.decoded_strings)) + res = ', label="#%s\\nchildren: %s%s"' % (self.instantiation_count, l, extra) + if self.state == MapBase.BLOCKED: + res += ", fillcolor=lightsalmon" + if self.state == MapBase.FRINGE: + res += ", fillcolor=lightgray" + if self.state == MapBase.PRELIMINARY: + res += ", fillcolor=lightslategray" return res def loads(space, w_s, w_errorcls=None): @@ -443,3 +1120,4 @@ raise OperationError(w_errorcls, w_e) finally: decoder.close() + diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py new file mode 100644 --- /dev/null +++ b/pypy/module/_pypyjson/simd.py @@ -0,0 +1,218 @@ +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import objectmodel, unroll +from rpython.rlib.rarithmetic import r_uint, intmask, LONG_BIT +from rpython.jit.backend.detect_cpu import autodetect + +# accelerators for string operations using simd on regular word sizes (*not* +# SSE instructions). this style is sometimes called SWAR (SIMD Within A +# Register) or "broadword techniques" + +# XXX remove wordsize and endianness restrictions properly, so far only x86-64 +# is tested + +USE_SIMD = False +if LONG_BIT == 64: + WORD_SIZE = 8 + EVERY_BYTE_ONE = 0x0101010101010101 + EVERY_BYTE_HIGHEST_BIT = 0x8080808080808080 + if autodetect() == "x86-64": + USE_SIMD = True +else: + WORD_SIZE = 4 + EVERY_BYTE_ONE = 0x01010101 + EVERY_BYTE_HIGHEST_BIT = 0x80808080 + + +# helpers + +unrolling_wordsize = unroll.unrolling_iterable(range(WORD_SIZE)) + +def char_repeated_word_width(ch): + return r_uint(EVERY_BYTE_ONE) * ord(ch) + +def any_char_zero(word): + return (word - r_uint(EVERY_BYTE_ONE)) & ~word & r_uint(EVERY_BYTE_HIGHEST_BIT) + +def any_char_in_words_zero(*words): + return _any_char_in_any_word_zero_accum(0, *words) + +def _any_char_in_any_word_zero_accum(accum, word, *words): + accum |= (word - r_uint(EVERY_BYTE_ONE)) & ~word + if not words: + return accum & r_uint(EVERY_BYTE_HIGHEST_BIT) + return _any_char_in_any_word_zero_accum(accum, *words) + +def print_chars(word): + # for debugging + out = '' + for i in range(WORD_SIZE): + out += chr(word & 0xff) + word >>= 8 + return out + +def index_nonzero(word): + # XXX can be done very cheap in theory + assert word + for i in unrolling_wordsize: + if word & 0xff: + return i + word >>= 8 + assert 0 + +def index_zero(word): + # XXX can be done very cheap in theory + assert any_char_zero(word) + for i in unrolling_wordsize: + if not word & 0xff: + return i + word >>= 8 + assert 0 # XXX ??? + +def splice_words(word, offset, other): + mask = ((~r_uint(0)) << (8 * offset)) + return (word & mask) | (other & ~mask) + + + + at objectmodel.always_inline +def position_string_ender(word): + maskquote = char_repeated_word_width('"') + maskbackslash = char_repeated_word_width('\\') + maskx20 = char_repeated_word_width(chr(0xff - 0x1f)) + # x1 and x2 check for equality, if a byte is 0 the corresponding + # char is equal to " or \ + x1 = maskquote ^ word + x2 = maskbackslash ^ word + # x3 checks for char < 0x20, the byte is 0 in that case + x3 = maskx20 & word + return any_char_in_words_zero(x1, x2, x3) + + at objectmodel.always_inline +def find_end_of_string_simd_unaligned(ll_chars, startpos, length): + ch = ll_chars[startpos] + strhash = (ord(ch) << 7) ^ 0x345678 + + wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) + num_safe_reads = (length - startpos) // WORD_SIZE + + bits = 0 + for i in range(num_safe_reads): + word = wordarray[i] + cond = position_string_ender(word) + if cond: + break + bits |= word + strhash = intmask((1000003 * strhash) ^ intmask(word)) + else: + # didn't find end of string yet, look at remaining chars + word = 0 + shift = 0 + i = 0 + for i in range(num_safe_reads * WORD_SIZE + startpos, length + 1): + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + bits |= ord(ch) + word |= ord(ch) << shift + shift += 8 + if shift: + strhash = intmask((1000003 * strhash) ^ intmask(word)) + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + return strhash, nonascii, i + + # compute endposition + nonzero = index_nonzero(cond) + endposition = startpos + i * WORD_SIZE + nonzero + if nonzero: + word = splice_words(r_uint(0), nonzero, word) + bits |= word + strhash = intmask((1000003 * strhash) ^ intmask(word)) + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + + return strhash, nonascii, endposition + + at objectmodel.always_inline +def find_end_of_string_simd_unaligned_no_hash(ll_chars, startpos, length): + ch = ll_chars[startpos] + + wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) + num_safe_reads = (length - startpos) // WORD_SIZE + + bits = 0 + for i in range(num_safe_reads): + word = wordarray[i] + cond = position_string_ender(word) + if cond: + break + bits |= word + else: + # didn't find end of string yet, look at remaining chars + word = 0 + shift = 0 + i = 0 + for i in range(num_safe_reads * WORD_SIZE + startpos, length + 1): + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + bits |= ord(ch) + word |= ord(ch) << shift + shift += WORD_SIZE + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + return nonascii, i + + # compute endposition + nonzero = index_nonzero(cond) + endposition = startpos + i * WORD_SIZE + nonzero + if nonzero: + word = splice_words(r_uint(0), nonzero, word) + bits |= word + + nonascii = bool(bits & char_repeated_word_width(chr(0x80))) + + return nonascii, endposition + + + at objectmodel.always_inline +def find_end_of_string_slow(ll_chars, i, length): + ch = ll_chars[i] + strhash = (ord(ch) << 7) ^ 0x345678 + word = 0 + shift = 0 + + bits = 0 + + while True: + # this loop is a fast path for strings which do not contain escape + # characters + ch = ll_chars[i] + if ch == '"' or ch == '\\' or ch < '\x20': + break + i += 1 + bits |= ord(ch) + + word |= ord(ch) << shift + shift += WORD_SIZE + if shift == WORD_SIZE * WORD_SIZE: + strhash = intmask((1000003 * strhash) ^ word) + shift = 0 + word = 0 + + if shift: + strhash = intmask((1000003 * strhash) ^ word) + return strhash, bool(bits & 0x80), i + +if USE_SIMD: + find_end_of_string = find_end_of_string_simd_unaligned + find_end_of_string_no_hash = find_end_of_string_simd_unaligned_no_hash +else: + find_end_of_string = find_end_of_string_slow + + @objectmodel.always_inline + def find_end_of_string_no_hash(ll_chars, i, length): + _, nonascii, i = find_end_of_string_slow(ll_chars, i, length) + return (nonascii, i) + + diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -1,31 +1,251 @@ # -*- encoding: utf-8 -*- -from pypy.module._pypyjson.interp_decoder import JSONDecoder +import pytest +from pypy.module._pypyjson.interp_decoder import JSONDecoder, Terminator, MapBase +from rpython.rtyper.lltypesystem import lltype, rffi -def test_skip_whitespace(): - s = ' hello ' - dec = JSONDecoder('fake space', s) - assert dec.pos == 0 - assert dec.skip_whitespace(0) == 3 - assert dec.skip_whitespace(3) == 3 - assert dec.skip_whitespace(8) == len(s) - dec.close() -class FakeSpace(object): - def newutf8(self, s, l): - return s +class TestJson(object): + def test_skip_whitespace(self): + s = ' hello ' + dec = JSONDecoder(self.space, s) + assert dec.pos == 0 + assert dec.skip_whitespace(0) == 3 + assert dec.skip_whitespace(3) == 3 + assert dec.skip_whitespace(8) == len(s) + dec.close() -def test_decode_key(): - s1 = "123" * 100 - s = ' "%s" "%s" ' % (s1, s1) - dec = JSONDecoder(FakeSpace(), s) - assert dec.pos == 0 - x = dec.decode_key(0) - assert x == s1 - # check caching - y = dec.decode_key(dec.pos) - assert y == s1 - assert y is x - dec.close() + def test_json_map(self): + m = Terminator(self.space) + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + m1 = m.get_next(w_a, '"a"', 0, 3, m) + assert m1.w_key == w_a + assert m1.single_nextmap is None + assert m1.key_repr == '"a"' + assert m1.key_repr_cmp('"a": 123', 0) + assert not m1.key_repr_cmp('b": 123', 0) + assert m.single_nextmap.w_key == w_a + + m2 = m.get_next(w_a, '"a"', 0, 3, m) + assert m2 is m1 + + m3 = m.get_next(w_b, '"b"', 0, 3, m) + assert m3.w_key == w_b + assert m3.single_nextmap is None + assert m3.key_repr == '"b"' + assert m.single_nextmap is m1 + + m4 = m3.get_next(w_c, '"c"', 0, 3, m) + assert m4.w_key == w_c + assert m4.single_nextmap is None + assert m4.key_repr == '"c"' + assert m3.single_nextmap is m4 + + def test_json_map_get_index(self): + m = Terminator(self.space) + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + m1 = m.get_next(w_a, 'a"', 0, 2, m) + assert m1.get_index(w_a) == 0 + assert m1.get_index(w_b) == -1 + + m2 = m.get_next(w_b, 'b"', 0, 2, m) + assert m2.get_index(w_b) == 0 + assert m2.get_index(w_a) == -1 + + m3 = m2.get_next(w_c, 'c"', 0, 2, m) + assert m3.get_index(w_b) == 0 + assert m3.get_index(w_c) == 1 + assert m3.get_index(w_a) == -1 + + def test_decode_key_map(self): + m = Terminator(self.space) + m_diff = Terminator(self.space) + for s1 in ["abc", "1001" * 10, u"ä".encode("utf-8")]: + s = ' "%s" "%s" "%s"' % (s1, s1, s1) + dec = JSONDecoder(self.space, s) + assert dec.pos == 0 + m1 = dec.decode_key_map(dec.skip_whitespace(0), m) + assert m1.w_key._utf8 == s1 + assert m1.key_repr == '"%s"' % s1 + + # check caching on w_key level + m2 = dec.decode_key_map(dec.skip_whitespace(dec.pos), m_diff) + assert m1.w_key is m2.w_key + + # check caching on map level + m3 = dec.decode_key_map(dec.skip_whitespace(dec.pos), m_diff) + assert m3 is m2 + dec.close() + + def test_decode_string_caching(self): + for s1 in ["abc", u"ä".encode("utf-8")]: + s = '"%s" "%s" "%s"' % (s1, s1, s1) + dec = JSONDecoder(self.space, s) + dec.MIN_SIZE_FOR_STRING_CACHE = 0 + assert dec.pos == 0 + w_x = dec.decode_string(1) + w_y = dec.decode_string(dec.skip_whitespace(dec.pos) + 1) + assert w_x is not w_y + # check caching + w_z = dec.decode_string(dec.skip_whitespace(dec.pos) + 1) + assert w_z is w_y + dec.close() + + def _make_some_maps(self): + # base -> m1 -> m2 -> m3 + # \-> m4 + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + w_d = self.space.newutf8("d", 1) + base = Terminator(self.space) + base.instantiation_count = 6 + m1 = base.get_next(w_a, 'a"', 0, 2, base) + m2 = m1.get_next(w_b, 'b"', 0, 2, base) + m3 = m2.get_next(w_c, 'c"', 0, 2, base) + m4 = m2.get_next(w_d, 'd"', 0, 2, base) + return base, m1, m2, m3, m4 + + # unit tests for map state transistions + def test_fringe_to_useful(self): + base, m1, m2, m3, m4 = self._make_some_maps() + base.instantiation_count = 6 + assert m1.state == MapBase.FRINGE + m1.instantiation_count = 6 + + assert m2.state == MapBase.PRELIMINARY + m2.instantiation_count = 6 + + assert m3.state == MapBase.PRELIMINARY + m3.instantiation_count = 2 + assert m2.single_nextmap is m3 + + assert m4.state == MapBase.PRELIMINARY + m4.instantiation_count = 4 + + m1.mark_useful(base) + assert m1.state == MapBase.USEFUL + assert m2.state == MapBase.USEFUL + assert m3.state == MapBase.FRINGE + assert m4.state == MapBase.USEFUL + assert m2.single_nextmap is m4 + + assert m1.number_of_leaves == 2 + base._check_invariants() + + def test_number_of_leaves(self): + w_x = self.space.newutf8("x", 1) + base, m1, m2, m3, m4 = self._make_some_maps() + assert base.number_of_leaves == 2 + assert m1.number_of_leaves == 2 + assert m2.number_of_leaves == 2 + assert m3.number_of_leaves == 1 + assert m4.number_of_leaves == 1 + m5 = m2.get_next(w_x, 'x"', 0, 2, base) + assert base.number_of_leaves == 3 + assert m1.number_of_leaves == 3 + assert m2.number_of_leaves == 3 + assert m5.number_of_leaves == 1 + + def test_number_of_leaves_after_mark_blocked(self): + w_x = self.space.newutf8("x", 1) + base, m1, m2, m3, m4 = self._make_some_maps() + m5 = m2.get_next(w_x, 'x"', 0, 2, base) + assert base.number_of_leaves == 3 + m2.mark_blocked(base) + assert base.number_of_leaves == 1 + + def test_mark_useful_cleans_fringe(self): + base, m1, m2, m3, m4 = self._make_some_maps() + base.instantiation_count = 6 + assert m1.state == MapBase.FRINGE + m1.instantiation_count = 6 + m2.instantiation_count = 6 + m3.instantiation_count = 2 + m4.instantiation_count = 4 + assert base.current_fringe == {m1: None} + + m1.mark_useful(base) + assert base.current_fringe == {m3: None} + + def test_cleanup_fringe(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + w_d = self.space.newutf8("d", 1) + base = Terminator(self.space) + base.instantiation_count = 6 + m1 = base.get_next(w_a, 'a"', 0, 2, base) + m2 = base.get_next(w_b, 'b"', 0, 2, base) + m3 = base.get_next(w_c, 'c"', 0, 2, base) + m4 = base.get_next(w_d, 'd"', 0, 2, base) + m5 = m4.get_next(w_a, 'a"', 0, 2, base) + base.instantiation_count = 7 + m1.instantiation_count = 2 + m2.instantiation_count = 2 + m3.instantiation_count = 2 + m4.instantiation_count = 1 + m5.instantiation_count = 1 + assert base.current_fringe == dict.fromkeys([m1, m2, m3, m4]) + + base.cleanup_fringe() + assert base.current_fringe == dict.fromkeys([m1, m2, m3]) + assert m4.state == MapBase.BLOCKED + assert m4.single_nextmap is None + assert m4.all_next is None + assert m5.state == MapBase.BLOCKED + assert m5.single_nextmap is None + assert m5.all_next is None + + def test_deal_with_blocked(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_c = self.space.newutf8("c", 1) + space = self.space + s = '{"a": 1, "b": 2, "c": 3}' + dec = JSONDecoder(space, s) + dec.startmap = base = Terminator(space) + m1 = base.get_next(w_a, 'a"', 0, 2, base) + m2 = m1.get_next(w_b, 'b"', 0, 2, base) + m2.mark_blocked(base) + w_res = dec.decode_object(1) + assert space.int_w(space.len(w_res)) == 3 + assert space.int_w(space.getitem(w_res, w_a)) == 1 + assert space.int_w(space.getitem(w_res, w_b)) == 2 + assert space.int_w(space.getitem(w_res, w_c)) == 3 + dec.close() + + def test_deal_with_blocked_number_of_leaves(self): + w_a = self.space.newutf8("a", 1) + w_b = self.space.newutf8("b", 1) + w_x = self.space.newutf8("x", 1) + w_u = self.space.newutf8("u", 1) + space = self.space + base = Terminator(space) + m1 = base.get_next(w_a, 'a"', 0, 2, base) + m2 = m1.get_next(w_b, 'b"', 0, 2, base) + m2.get_next(w_x, 'x"', 0, 2, base) + m2.get_next(w_u, 'u"', 0, 2, base) + assert base.number_of_leaves == 2 + m2.mark_blocked(base) + assert base.number_of_leaves == 1 + + def test_instatiation_count(self): + m = Terminator(self.space) + dec = JSONDecoder(self.space, '"abc" "def"') + m1 = dec.decode_key_map(dec.skip_whitespace(0), m) + m2 = dec.decode_key_map(dec.skip_whitespace(6), m1) + m1 = dec.decode_key_map(dec.skip_whitespace(0), m) + m2 = dec.decode_key_map(dec.skip_whitespace(6), m1) + m1 = dec.decode_key_map(dec.skip_whitespace(0), m) + + assert m1.instantiation_count == 3 + assert m2.instantiation_count == 2 + dec.close() + class AppTest(object): spaceconfig = {"usemodules": ['_pypyjson']} @@ -106,6 +326,12 @@ s = r'"\u1234"' assert _pypyjson.loads(s) == '\u1234' + def test_escape_sequence_mixed_with_unicode(self): + import _pypyjson + assert _pypyjson.loads(r'"abc\\' + u'ä"') == u'abc\\ä' + assert _pypyjson.loads(r'"abc\"' + u'ä"') == u'abc"ä' + assert _pypyjson.loads(r'"def\u1234' + u'ä"') == u'def\u1234ä' + def test_invalid_utf_8(self): import _pypyjson s = '"\xe0"' # this is an invalid UTF8 sequence inside a string @@ -175,9 +401,14 @@ s = '{"hello": "world", "aaa": "bbb"}' assert _pypyjson.loads(s) == {'hello': 'world', 'aaa': 'bbb'} + assert _pypyjson.loads(s) == {'hello': 'world', + 'aaa': 'bbb'} raises(ValueError, _pypyjson.loads, '{"key"') raises(ValueError, _pypyjson.loads, '{"key": 42') + assert _pypyjson.loads('{"neighborhood": ""}') == { + "neighborhood": ""} + def test_decode_object_nonstring_key(self): import _pypyjson raises(ValueError, "_pypyjson.loads('{42: 43}')") @@ -248,11 +479,11 @@ def test_error_position(self): import _pypyjson test_cases = [ - ('[,', "Unexpected ',' at", 1), - ('{"spam":[}', "Unexpected '}' at", 9), + ('[,', "Unexpected ','", 1), + ('{"spam":[}', "Unexpected '}'", 9), ('[42:', "Unexpected ':' when decoding array", 3), ('[42 "spam"', "Unexpected '\"' when decoding array", 4), - ('[42,]', "Unexpected ']' at", 4), + ('[42,]', "Unexpected ']'", 4), ('{"spam":[42}', "Unexpected '}' when decoding array", 11), ('["]', 'Unterminated string starting at', 1), ('["spam":', "Unexpected ':' when decoding array", 7), @@ -275,4 +506,4 @@ class MyError(Exception): pass exc = raises(MyError, _pypyjson.loads, 'nul', MyError) - assert exc.value.args == ('Error when decoding null at', 'nul', 1) + assert exc.value.args == ('Error when decoding null', 'nul', 1) diff --git a/pypy/module/_pypyjson/test/test_simd.py b/pypy/module/_pypyjson/test/test_simd.py new file mode 100644 --- /dev/null +++ b/pypy/module/_pypyjson/test/test_simd.py @@ -0,0 +1,107 @@ +import sys +import pytest +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import r_uint, intmask + +from pypy.module._pypyjson.simd import USE_SIMD +from pypy.module._pypyjson.simd import find_end_of_string_slow +from pypy.module._pypyjson.simd import print_chars +from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned, WORD_SIZE +from pypy.module._pypyjson.simd import find_end_of_string_simd_unaligned_no_hash + +try: + from hypothesis import example, given, strategies +except ImportError: + pytest.skip("missing hypothesis!") + +if not USE_SIMD: + pytest.skip("only implemented for 64 bit for now") + +def fill_to_word_size(res, ch=" "): + if len(res) % WORD_SIZE != 0: + res += ch * (WORD_SIZE - (len(res) % WORD_SIZE)) + return res + +def string_to_word(s): + assert len(s) == WORD_SIZE + ll_chars, flag = rffi.get_nonmovingbuffer_final_null(s) + try: + wordarray = rffi.cast(rffi.ULONGP, ll_chars) + return wordarray[0] + finally: + rffi.free_nonmovingbuffer(s, ll_chars, flag) + +def ll(callable, string, *args): + ll_chars, flag = rffi.get_nonmovingbuffer_final_null(string) + try: + return callable(ll_chars, *args) + finally: + rffi.free_nonmovingbuffer(string, ll_chars, flag) + +word = strategies.builds( + r_uint, strategies.integers(min_value=-sys.maxint-1, max_value=sys.maxint)) + +def build_string(prefix, content, end, suffix): + res = prefix + '"' + "".join([chr(x) for x in content]) + end + suffix + return fill_to_word_size(res), len(prefix) + 1 + +string_in_context_strategy = strategies.builds( + build_string, prefix=strategies.binary(), + content=strategies.lists(strategies.integers(1, 255), min_size=1), + end=strategies.sampled_from('"\\\x00\x01'), + suffix=strategies.binary()) + +def compare(string, res1, res2): + hash1, nonascii1, endindex1 = res1 + hash2, nonascii2, endindex2 = res2 + assert endindex1 == endindex2 + if string[endindex1 - 1] == '"': + assert hash1 == hash2 + assert nonascii1 == nonascii2 + + + at example(('" \x80" ', 1)) + at example(('"\x01" ', 1)) + at example(('"aaaaaaaa"\x00\x00\x00\x00\x00\x00\x00 ', 1)) + at example(('"aaaaaaaa" ', 1)) + at example(('"12"', 1)) + at example(('"1234567abcdefghAB"', 1)) + at example(('"1234567abcdefgh"', 1)) + at example((' "123456ABCDEF" \x00', 2)) + at example((' "123456aaaaaaaaABCDEF"\x00', 2)) + at given(string_in_context_strategy) +def test_find_end_of_string(a): + (string, startindex) = a + res = ll(find_end_of_string_slow, string, startindex, len(string)) + hash, nonascii1, endposition1 = res + ch = string[endposition1] + assert ch == '"' or ch == '\\' or ch < '\x20' + for ch in string[startindex:endposition1]: + assert not (ch == '"' or ch == '\\' or ch < '\x20') + compare(string, res, ll(find_end_of_string_simd_unaligned, string, startindex, len(string))) + + nonascii2, endposition2 = ll(find_end_of_string_simd_unaligned_no_hash, string, startindex, len(string)) + assert nonascii1 == nonascii2 + assert endposition1 == endposition2 + + at given(string_in_context_strategy, strategies.binary(min_size=1)) +def test_find_end_of_string_position_invariance(a, prefix): + fn = find_end_of_string_simd_unaligned + (string, startindex) = a + h1, nonascii1, i1 = ll(fn, string, startindex, len(string)) + string2 = prefix + string + h2, nonascii2, i2 = ll(fn, string2, startindex + len(prefix), len(string) + len(prefix)) + assert h1 == h2 + assert nonascii1 == nonascii2 + assert i1 + len(prefix) == i2 + + at given(string_in_context_strategy, strategies.binary(min_size=1)) +def test_find_end_of_string_position_invariance_no_hash(a, prefix): + fn = find_end_of_string_simd_unaligned_no_hash + (string, startindex) = a + nonascii1, i1 = ll(fn, string, startindex, len(string)) + string2 = prefix + string + nonascii2, i2 = ll(fn, string2, startindex + len(prefix), len(string) + len(prefix)) + assert nonascii1 == nonascii2 + assert i1 + len(prefix) == i2 + diff --git a/pypy/objspace/std/jsondict.py b/pypy/objspace/std/jsondict.py new file mode 100644 --- /dev/null +++ b/pypy/objspace/std/jsondict.py @@ -0,0 +1,167 @@ +"""dict implementation specialized for object loaded by the _pypyjson module. + +Somewhat similar to MapDictStrategy, also uses a map. +""" + +from rpython.rlib import jit, rerased, objectmodel, debug + +from pypy.objspace.std.dictmultiobject import ( + UnicodeDictStrategy, DictStrategy, + create_iterator_classes, W_DictObject) + + +def from_values_and_jsonmap(space, values_w, jsonmap): + if not objectmodel.we_are_translated(): + assert len(values_w) == len(jsonmap.get_keys_in_order()) + assert len(values_w) != 0 + debug.make_sure_not_resized(values_w) + strategy = jsonmap.strategy_instance + if strategy is None: + jsonmap.strategy_instance = strategy = JsonDictStrategy(space, jsonmap) + storage = strategy.erase(values_w) + return W_DictObject(space, strategy, storage) + +def devolve_jsonmap_dict(w_dict): + assert isinstance(w_dict, W_DictObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, JsonDictStrategy) + strategy.switch_to_unicode_strategy(w_dict) + +def get_jsonmap_from_dict(w_dict): + assert isinstance(w_dict, W_DictObject) + strategy = w_dict.get_strategy() + assert isinstance(strategy, JsonDictStrategy) + return strategy.jsonmap + +class JsonDictStrategy(DictStrategy): + erase, unerase = rerased.new_erasing_pair("jsondict") + erase = staticmethod(erase) + unerase = staticmethod(unerase) + + _immutable_fields_ = ['jsonmap'] + + def __init__(self, space, jsonmap): + DictStrategy.__init__(self, space) + self.jsonmap = jsonmap + + def wrap(self, w_key): + return w_key + + def wrapkey(space, key): + return key + + def get_empty_storage(self): + raise NotImplementedError("should not be reachable") + + def is_correct_type(self, w_obj): + space = self.space + return space.is_w(space.type(w_obj), space.w_unicode) + + def _never_equal_to(self, w_lookup_type): + return False + + def length(self, w_dict): + return len(self.unerase(w_dict.dstorage)) + + def getitem(self, w_dict, w_key): + if self.is_correct_type(w_key): + return self.getitem_unicode(w_dict, w_key) + else: + self.switch_to_unicode_strategy(w_dict) + return w_dict.getitem(w_key) + + def getitem_unicode(self, w_dict, w_key): + storage_w = self.unerase(w_dict.dstorage) + if jit.isconstant(w_key): + jit.promote(self) + index = self.jsonmap.get_index(w_key) + if index == -1: + return None + return storage_w[index] + + def setitem(self, w_dict, w_key, w_value): + if self.is_correct_type(w_key): + storage_w = self.unerase(w_dict.dstorage) + index = self.jsonmap.get_index(w_key) + if index != -1: + storage_w[index] = w_value + return + self.switch_to_unicode_strategy(w_dict) + w_dict.setitem(w_key, w_value) + + def setdefault(self, w_dict, w_key, w_default): + if self.is_correct_type(w_key): + w_result = self.getitem_unicode(w_dict, w_key) + if w_result is not None: + return w_result + self.switch_to_unicode_strategy(w_dict) + return w_dict.setdefault(w_key, w_default) + + def delitem(self, w_dict, w_key): + self.switch_to_unicode_strategy(w_dict) + return w_dict.delitem(w_key) + + def popitem(self, w_dict): + self.switch_to_unicode_strategy(w_dict) + return w_dict.popitem() + + def switch_to_unicode_strategy(self, w_dict): + strategy = self.space.fromcache(UnicodeDictStrategy) + values_w = self.unerase(w_dict.dstorage) + storage = strategy.get_empty_storage() + d_new = strategy.unerase(storage) + keys_in_order = self.jsonmap.get_keys_in_order() + assert len(keys_in_order) == len(values_w) + for index, w_key in enumerate(keys_in_order): + assert w_key is not None + assert type(w_key) is self.space.UnicodeObjectCls + d_new[w_key] = values_w[index] + w_dict.set_strategy(strategy) + w_dict.dstorage = storage + + def w_keys(self, w_dict): + return self.space.newlist(self.jsonmap.get_keys_in_order()) + + def values(self, w_dict): + return self.unerase(w_dict.dstorage)[:] # to make resizable + + def items(self, w_dict): + space = self.space + storage_w = self.unerase(w_dict.dstorage) + res = [None] * len(storage_w) + for index, w_key in enumerate(self.jsonmap.get_keys_in_order()): + res[index] = space.newtuple([w_key, storage_w[index]]) + return res + + def getiterkeys(self, w_dict): + return iter(self.jsonmap.get_keys_in_order()) + + def getitervalues(self, w_dict): + storage_w = self.unerase(w_dict.dstorage) + return iter(storage_w) + + def getiteritems_with_hash(self, w_dict): + storage_w = self.unerase(w_dict.dstorage) + return ZipItemsWithHash(self.jsonmap.get_keys_in_order(), storage_w) + + +class ZipItemsWithHash(object): + def __init__(self, list1, list2): + assert len(list1) == len(list2) + self.list1 = list1 + self.list2 = list2 + self.i = 0 + + def __iter__(self): + return self + + def next(self): + i = self.i + if i >= len(self.list1): + raise StopIteration + self.i = i + 1 + w_key = self.list1[i] + return (w_key, self.list2[i], w_key.hash_w()) + + +create_iterator_classes(JsonDictStrategy) diff --git a/pypy/objspace/std/test/test_jsondict.py b/pypy/objspace/std/test/test_jsondict.py new file mode 100644 --- /dev/null +++ b/pypy/objspace/std/test/test_jsondict.py @@ -0,0 +1,89 @@ + +class AppTest(object): + spaceconfig = {"objspace.usemodules._pypyjson": True} + + def test_check_strategy(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1}') + assert __pypy__.strategy(d) == "JsonDictStrategy" + d = _pypyjson.loads('{}') + assert __pypy__.strategy(d) == "EmptyDictStrategy" + + def test_simple(self): + import __pypy__ + import _pypyjson + + d = _pypyjson.loads('{"a": 1, "b": "x"}') + assert len(d) == 2 From pypy.commits at gmail.com Tue Jun 11 07:26:33 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 11 Jun 2019 04:26:33 -0700 (PDT) Subject: [pypy-commit] pypy arm64: start fighting registers Message-ID: <5cff8fe9.1c69fb81.70468.92f0@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96791:7f271fc65bd8 Date: 2019-06-11 11:25 +0000 http://bitbucket.org/pypy/pypy/changeset/7f271fc65bd8/ Log: start fighting registers diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -4,11 +4,9 @@ from rpython.jit.backend.aarch64.locations import imm, StackLocation, get_fp_offset #from rpython.jit.backend.arm.helper.regalloc import VMEM_imm_size from rpython.jit.backend.aarch64.opassembler import ResOpAssembler -from rpython.jit.backend.aarch64.regalloc import (Regalloc, +from rpython.jit.backend.aarch64.regalloc import (Regalloc, check_imm_arg, operations as regalloc_operations, guard_operations, comp_operations, CoreRegisterManager) -# CoreRegisterManager, check_imm_arg, VFPRegisterManager, -#from rpython.jit.backend.arm import callbuilder from rpython.jit.backend.aarch64 import registers as r from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.llsupport import jitframe @@ -288,21 +286,19 @@ self._push_all_regs_to_jitframe(mc, [], withfloats) if exc: - return # fix later - XXX # We might have an exception pending. Load it into r4 # (this is a register saved across calls) - mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) - mc.LDR_ri(r.r4.value, r.r5.value) + mc.gen_load_int(r.x5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.x4.value, r.x5.value, 0) # clear the exc flags - mc.gen_load_int(r.r6.value, 0) - mc.STR_ri(r.r6.value, r.r5.value) # pos_exc_value is still in r5 - mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) - mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.x6.value, 0) + mc.STR_ri(r.x6.value, r.x5.value, 0) # pos_exc_value is still in r5 + mc.gen_load_int(r.x5.value, self.cpu.pos_exception()) + mc.STR_ri(r.x6.value, r.x5.value, 0) # save r4 into 'jf_guard_exc' offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc') assert check_imm_arg(abs(offset)) - mc.STR_ri(r.r4.value, r.fp.value, imm=offset) + mc.STR_ri(r.x4.value, r.fp.value, offset) # now we return from the complete frame, which starts from # _call_header_with_stack_check(). The LEA in _call_footer below # throws away most of the frame, including all the PUSHes that we @@ -315,6 +311,12 @@ rawstart = mc.materialize(self.cpu, []) self.failure_recovery_code[exc + 2 * withfloats] = rawstart + def propagate_memoryerror_if_reg_is_null(self, reg_loc): + # see ../x86/assembler.py:genop_discard_check_memory_error() + self.mc.CMP_ri(reg_loc.value, 0) + self.mc.B_ofs_cond(6 * 4, c.NE) + self.mc.B(self.propagate_exception_path) + def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): pass # XXX @@ -365,7 +367,7 @@ if gcrootmap and gcrootmap.is_shadow_stack: self._load_shadowstack_top(mc, r.r5, gcrootmap) # store the new jitframe addr in the shadowstack - mc.STR_ri(r.r0.value, r.r5.value, imm=-WORD) + mc.STR_ri(r.x0.value, r.r5.value, imm=-WORD) # reset the jf_gcmap field in the jitframe mc.gen_load_int(r.ip0.value, 0) @@ -385,13 +387,60 @@ """ Resest the exception. If excvalloc is None, then store it on the frame in jf_guard_exc """ - pass + assert excvalloc is not r.ip0 + assert exctploc is not r.ip0 + tmpreg = r.lr + mc.gen_load_int(r.ip0.value, self.cpu.pos_exc_value()) + if excvalloc is not None: # store + assert excvalloc.is_core_reg() + self.load_reg(mc, excvalloc, r.ip0) + if on_frame: + # store exc_value in JITFRAME + ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + assert check_imm_arg(ofs) + # + self.load_reg(mc, r.ip0, r.ip0, helper=tmpreg) + # + self.store_reg(mc, r.ip0, r.fp, ofs, helper=tmpreg) + if exctploc is not None: + # store pos_exception in exctploc + assert exctploc.is_core_reg() + mc.gen_load_int(r.ip0.value, self.cpu.pos_exception()) + self.load_reg(mc, exctploc, r.ip0, helper=tmpreg) + + if on_frame or exctploc is not None: + mc.gen_load_int(r.ip0.value, self.cpu.pos_exc_value()) + + # reset exception + mc.gen_load_int(tmpreg.value, 0) + + self.store_reg(mc, tmpreg, r.ip0, 0) + + mc.gen_load_int(r.ip0.value, self.cpu.pos_exception()) + self.store_reg(mc, tmpreg, r.ip0, 0) def _restore_exception(self, mc, excvalloc, exctploc): pass def _build_propagate_exception_path(self): - pass + mc = InstrBuilder() + self._store_and_reset_exception(mc, r.x0) + ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + # make sure ofs fits into a register + assert check_imm_arg(ofs) + self.store_reg(mc, r.x0, r.fp, ofs) + propagate_exception_descr = rffi.cast(lltype.Signed, + cast_instance_to_gcref(self.cpu.propagate_exception_descr)) + # put propagate_exception_descr into frame + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + # make sure ofs fits into a register + assert check_imm_arg(ofs) + mc.gen_load_int(r.x0.value, propagate_exception_descr) + self.store_reg(mc, r.x0, r.fp, ofs) + mc.MOV_rr(r.x0.value, r.fp.value) + self.gen_func_epilog(mc) + rawstart = mc.materialize(self.cpu, []) + self.propagate_exception_path = rawstart def _build_cond_call_slowpath(self, supports_floats, callee_only): pass @@ -464,6 +513,7 @@ self.load_from_gc_table(r.ip0.value, faildescrindex) self.store_reg(self.mc, r.ip0, r.fp, WORD) self.push_gcmap(self.mc, gcmap=guardtok.gcmap) + assert target self.mc.BL(target) return startpos @@ -776,7 +826,7 @@ mc.RET_r(r.lr.value) - def store_reg(self, mc, source, base, ofs=0): + def store_reg(self, mc, source, base, ofs=0, helper=None): # uses r.ip1 as a temporary if source.is_vfp_reg(): return self._store_vfp_reg(mc, source, base, ofs) @@ -803,6 +853,20 @@ # mc.gen_load_int(r.ip1, ofs) # mc.STR_rr(source.value, base.value, r.ip1) + def load_reg(self, mc, target, base, ofs=0, helper=None): + if target.is_vfp_reg(): + return self._load_vfp_reg(mc, target, base, ofs) + elif target.is_core_reg(): + return self._load_core_reg(mc, target, base, ofs) + + def _load_core_reg(self, mc, target, base, ofs): + if check_imm_arg(abs(ofs)): + mc.LDR_ri(target.value, base.value, ofs) + else: + XXX + mc.gen_load_int(helper.value, ofs, cond=cond) + mc.LDR_rr(target.value, base.value, helper.value, cond=cond) + def check_frame_before_jump(self, target_token): if target_token in self.target_tokens_currently_compiling: return diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -281,7 +281,7 @@ def B(self, target): target = rffi.cast(lltype.Signed, target) self.gen_load_int_full(r.ip0.value, target) - self.BR_r(r.ip0.value) + self.BR_r(r.ip0.value) def BL(self, target): # XXX use the IMM version if close enough diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -193,6 +193,9 @@ self.mc.ADD_ri(value_loc.value, value_loc.value, 1) self.mc.STR_ri(value_loc.value, base_loc.value, 0) + def emit_op_check_memory_error(self, op, arglocs): + self.propagate_memoryerror_if_reg_is_null(arglocs[0]) + def _genop_same_as(self, op, arglocs): argloc, resloc = arglocs if argloc is not resloc: @@ -451,7 +454,18 @@ self.mc.LDR_ri(r.ip0.value, arglocs[0].value, offset) self.mc.gen_load_int_full(r.ip1.value, arglocs[1].value) self.mc.CMP_rr(r.ip0.value, r.ip1.value) - self._emit_guard(op, c.EQ, arglocs[2:]) + self._emit_guard(op, c.EQ, arglocs[2:]) + + def emit_op_guard_exception(self, op, arglocs): + loc, resloc, pos_exc_value, pos_exception = arglocs[:4] + failargs = arglocs[4:] + self.mc.gen_load_int(r.ip1.value, pos_exception.value) + self.mc.LDR_ri(r.ip0.value, r.ip1.value, 0) + + self.mc.CMP_rr(r.ip0.value, loc.value) + self._emit_guard(op, c.EQ, failargs) + self._store_and_reset_exception(self.mc, resloc) + # ----------------------------- call ------------------------------ diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -695,6 +695,20 @@ prepare_op_guard_gc_type = prepare_op_guard_class prepare_op_guard_subclass = prepare_op_guard_class + def prepare_op_guard_exception(self, op): + boxes = op.getarglist() + arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) + loc = self.make_sure_var_in_reg(arg0) + if op in self.longevity: + resloc = self.force_allocate_reg(op, boxes) + self.possibly_free_var(op) + else: + resloc = None + pos_exc_value = imm(self.cpu.pos_exc_value()) + pos_exception = imm(self.cpu.pos_exception()) + arglocs = [loc, resloc, pos_exc_value, pos_exception] + self._guard_impl(op) + return arglocs + prepare_op_ptr_eq = prepare_op_instance_ptr_eq = prepare_op_int_eq prepare_op_ptr_ne = prepare_op_instance_ptr_ne = prepare_op_int_ne @@ -730,6 +744,10 @@ res = self.force_allocate_reg(op) return [arg0, arg1, args[2], args[3], res] + def prepare_op_check_memory_error(self, op): + argloc = self.make_sure_var_in_reg(op.getarg(0)) + return [argloc] + def prepare_op_jump(self, op): assert self.jump_target_descr is None descr = op.getdescr() diff --git a/rpython/jit/backend/aarch64/registers.py b/rpython/jit/backend/aarch64/registers.py --- a/rpython/jit/backend/aarch64/registers.py +++ b/rpython/jit/backend/aarch64/registers.py @@ -23,4 +23,5 @@ callee_saved_registers = [] # x19, x20, x21, x22] -argument_regs = caller_resp = [x0, x1, x2, x3, x4, x5, x6, x7] +argument_regs = [x0, x1, x2, x3, x4, x5, x6, x7] +caller_resp = argument_regs + [x8, x9, x10, x11, x12, x13, x14, x15] \ No newline at end of file From pypy.commits at gmail.com Tue Jun 11 07:26:35 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 11 Jun 2019 04:26:35 -0700 (PDT) Subject: [pypy-commit] pypy arm64: start fighting exceptions Message-ID: <5cff8feb.1c69fb81.c6976.3ecc@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96792:17ca4ffd0a3f Date: 2019-06-11 11:25 +0000 http://bitbucket.org/pypy/pypy/changeset/17ca4ffd0a3f/ Log: start fighting exceptions diff --git a/rpython/jit/backend/aarch64/registers.py b/rpython/jit/backend/aarch64/registers.py --- a/rpython/jit/backend/aarch64/registers.py +++ b/rpython/jit/backend/aarch64/registers.py @@ -24,4 +24,4 @@ callee_saved_registers = [] # x19, x20, x21, x22] argument_regs = [x0, x1, x2, x3, x4, x5, x6, x7] -caller_resp = argument_regs + [x8, x9, x10, x11, x12, x13, x14, x15] \ No newline at end of file +caller_resp = argument_regs + [x8, x9, x10, x11, x12, x13, x14, x15] From pypy.commits at gmail.com Tue Jun 11 07:32:15 2019 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 11 Jun 2019 04:32:15 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps-py3.6: test and fix Message-ID: <5cff913f.1c69fb81.bced1.e5f1@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps-py3.6 Changeset: r96793:36f4881e7ef5 Date: 2019-06-11 13:31 +0200 http://bitbucket.org/pypy/pypy/changeset/36f4881e7ef5/ Log: test and fix diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -280,8 +280,8 @@ def _raise_object_error(self, ch, start, i): if ch == '\0': self._raise("Unterminated object starting at", start) - else: - self._raise("Unexpected '%s' when decoding object (char %d)" % ch, i) + else:?!?jedi=0, ?!? (*_*param msg*_*, param pos) ?!?jedi?!? + self._raise("Unexpected '%s' when decoding object" % ch, i) def decode_array(self, i): """ Decode a list. i must be after the opening '[' """ diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -488,6 +488,7 @@ ('["]', 'Unterminated string starting at', 1), ('["spam":', "Unexpected ':' when decoding array", 7), ('[{]', "Key name must be string at char", 2), + ('{"a": 1 "b": 2}', "Unexpected '\"' when decoding object", 8), ] for inputtext, errmsg, errpos in test_cases: exc = raises(ValueError, _pypyjson.loads, inputtext) From pypy.commits at gmail.com Tue Jun 11 10:19:43 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 11 Jun 2019 07:19:43 -0700 (PDT) Subject: [pypy-commit] pypy arm64: implement _restore_exception Message-ID: <5cffb87f.1c69fb81.c37f7.9965@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96794:ce1202a8787b Date: 2019-06-11 14:18 +0000 http://bitbucket.org/pypy/pypy/changeset/ce1202a8787b/ Log: implement _restore_exception diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -389,7 +389,7 @@ """ assert excvalloc is not r.ip0 assert exctploc is not r.ip0 - tmpreg = r.lr + tmpreg = r.ip1 mc.gen_load_int(r.ip0.value, self.cpu.pos_exc_value()) if excvalloc is not None: # store assert excvalloc.is_core_reg() @@ -420,7 +420,25 @@ self.store_reg(mc, tmpreg, r.ip0, 0) def _restore_exception(self, mc, excvalloc, exctploc): - pass + assert excvalloc is not r.ip0 + assert exctploc is not r.ip0 + mc.gen_load_int(r.ip0.value, self.cpu.pos_exc_value()) + if excvalloc is not None: + assert excvalloc.is_core_reg() + self.store_reg(mc, excvalloc, r.ip0) + else: + assert exctploc is not r.fp + # load exc_value from JITFRAME and put it in pos_exc_value + ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + self.load_reg(mc, r.ip1, r.fp, ofs) + self.store_reg(mc, r.ip1, r.ip0) + # reset exc_value in the JITFRAME + mc.gen_load_int(r.ip1.value, 0) + self.store_reg(mc, r.ip1, r.fp, ofs) + + # restore pos_exception from exctploc register + mc.gen_load_int(r.ip0.value, self.cpu.pos_exception()) + self.store_reg(mc, exctploc, r.ip0) def _build_propagate_exception_path(self): mc = InstrBuilder() diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -10,6 +10,7 @@ from rpython.jit.backend.llsupport.gcmap import allocate_gcmap from rpython.jit.backend.llsupport.regalloc import get_scale from rpython.jit.metainterp.history import TargetToken +from rpython.jit.metainterp.resoperation import rop def gen_comp_op(name, flag): def emit_op(self, op, arglocs): @@ -465,7 +466,32 @@ self.mc.CMP_rr(r.ip0.value, loc.value) self._emit_guard(op, c.EQ, failargs) self._store_and_reset_exception(self.mc, resloc) - + + def emit_op_guard_no_exception(self, op, arglocs): + loc = arglocs[0] + failargs = arglocs[1:] + self.mc.LDR_ri(loc.value, loc.value, 0) + self.mc.CMP_ri(loc.value, 0) + self._emit_guard(op, c.EQ, failargs) + # If the previous operation was a COND_CALL, overwrite its conditional + # jump to jump over this GUARD_NO_EXCEPTION as well, if we can + #if self._find_nearby_operation(-1).getopnum() == rop.COND_CALL: + # XXX + # jmp_adr, prev_cond = self.previous_cond_call_jcond + # pmc = OverwritingBuilder(self.mc, jmp_adr, WORD) + # pmc.B_offs(self.mc.currpos(), prev_cond) + + def emit_op_save_exc_class(self, op, arglocs): + resloc = arglocs[0] + self.mc.gen_load_int(r.ip0.value, self.cpu.pos_exception()) + self.load_reg(self.mc, resloc, r.ip0) + + def emit_op_save_exception(self, op, arglocs): + resloc = arglocs[0] + self._store_and_reset_exception(self.mc, resloc) + + def emit_op_restore_exception(self, op, arglocs): + self._restore_exception(self.mc, arglocs[1], arglocs[0]) # ----------------------------- call ------------------------------ diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -20,6 +20,8 @@ from rpython.jit.backend.llsupport.descr import CallDescr from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.rlib.rarithmetic import r_uint + class TempInt(TempVar): @@ -709,6 +711,21 @@ arglocs = [loc, resloc, pos_exc_value, pos_exception] + self._guard_impl(op) return arglocs + def prepare_op_guard_no_exception(self, op): + loc = self.make_sure_var_in_reg(ConstInt(self.cpu.pos_exception())) + return [loc] + self._guard_impl(op) + + def prepare_op_save_exception(self, op): + resloc = self.force_allocate_reg(op) + return [resloc] + prepare_op_save_exc_class = prepare_op_save_exception + + def prepare_op_restore_exception(self, op): + boxes = op.getarglist() + loc0 = self.make_sure_var_in_reg(op.getarg(0), boxes) # exc class + loc1 = self.make_sure_var_in_reg(op.getarg(1), boxes) # exc instance + return [loc0, loc1] + prepare_op_ptr_eq = prepare_op_instance_ptr_eq = prepare_op_int_eq prepare_op_ptr_ne = prepare_op_instance_ptr_ne = prepare_op_int_ne From pypy.commits at gmail.com Tue Jun 11 15:08:19 2019 From: pypy.commits at gmail.com (cfbolz) Date: Tue, 11 Jun 2019 12:08:19 -0700 (PDT) Subject: [pypy-commit] pypy json-decoder-maps-py3.6: pfff, too much vim magic Message-ID: <5cfffc23.1c69fb81.5fe14.9234@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: json-decoder-maps-py3.6 Changeset: r96795:74f2f7e4c35a Date: 2019-06-11 21:07 +0200 http://bitbucket.org/pypy/pypy/changeset/74f2f7e4c35a/ Log: pfff, too much vim magic diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -280,7 +280,7 @@ def _raise_object_error(self, ch, start, i): if ch == '\0': self._raise("Unterminated object starting at", start) - else:?!?jedi=0, ?!? (*_*param msg*_*, param pos) ?!?jedi?!? + else: self._raise("Unexpected '%s' when decoding object" % ch, i) def decode_array(self, i): From pypy.commits at gmail.com Wed Jun 12 07:50:52 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 12 Jun 2019 04:50:52 -0700 (PDT) Subject: [pypy-commit] pypy default: backport rlib socket changes from 440feb6ea372 (minus tabs) Message-ID: <5d00e71c.1c69fb81.77d2f.35c4@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r96797:8377609877a4 Date: 2019-06-12 13:48 +0200 http://bitbucket.org/pypy/pypy/changeset/8377609877a4/ Log: backport rlib socket changes from 440feb6ea372 (minus tabs) diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -203,6 +203,7 @@ FD_CONNECT_BIT FD_CLOSE_BIT WSA_IO_PENDING WSA_IO_INCOMPLETE WSA_INVALID_HANDLE WSA_INVALID_PARAMETER WSA_NOT_ENOUGH_MEMORY WSA_OPERATION_ABORTED +WSA_FLAG_OVERLAPPED SIO_RCVALL SIO_KEEPALIVE_VALS SIOCGIFNAME SIOCGIFINDEX @@ -1010,6 +1011,7 @@ CConfig.WSAPROTOCOL_INFO = platform.Struct( 'WSAPROTOCOL_INFOA', []) # Struct is just passed between functions + CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( 'FROM_PROTOCOL_INFO') @@ -1035,6 +1037,45 @@ ('keepalivetime', rffi.ULONG), ('keepaliveinterval', rffi.ULONG)]) + CConfig.GUID = platform.Struct( + 'struct _GUID', + [('Data1', rffi.UINT), + ('Data2', rffi.UINT), + ('Data3', rffi.UINT), + ('Data4', rffi.CFixedArray(rffi.UCHAR, 8)) + ]) + + CConfig.WSAPROTOCOLCHAIN = platform.Struct( + 'struct _WSAPROTOCOLCHAIN', + [('ChainLen', rffi.INT), + ('ChainEntries', rffi.CFixedArray(rffi.UINT, 7))]) + + WSAPROTOCOLCHAIN = CConfig.WSAPROTOCOLCHAIN + GUID = CConfig.GUID + + CConfig.WSAPROTOCOL_INFOW = platform.Struct( + 'struct _WSAPROTOCOL_INFOW', + [('dwServiceFlags1', rffi.UINT), + ('dwServiceFlags2', rffi.UINT), + ('dwServiceFlags3', rffi.UINT), + ('dwServiceFlags4', rffi.UINT), + ('dwProviderFlags', rffi.UINT), + ('ProviderId', GUID), + ('dwCatalogEntryId', rffi.UINT), + ('ProtocolChain', WSAPROTOCOLCHAIN), + ('iVersion', rffi.INT), + ('iAddressFamily', rffi.INT), + ('iMaxSockAddr', rffi.INT), + ('iMinSockAddr', rffi.INT), + ('iSocketType', rffi.INT), + ('iProtocol', rffi.INT), + ('iProtocolMaxOffset', rffi.INT), + ('iNetworkByteOrder', rffi.INT), + ('iSecurityScheme', rffi.INT), + ('dwMessageSize', rffi.UINT), + ('dwProviderReserved', rffi.UINT), + ('szProtocol', rffi.CFixedArray(rffi.UCHAR, 256))]) + class cConfig: pass @@ -1337,6 +1378,20 @@ rffi.VOIDP, rwin32.DWORD, rwin32.LPDWORD, rffi.VOIDP, rffi.VOIDP], rffi.INT, save_err=SAVE_ERR) + + WSAPROTOCOL_INFOW = cConfig.WSAPROTOCOL_INFOW + + WSADuplicateSocketW = external('WSADuplicateSocketW', + [socketfd_type, rwin32.DWORD, + lltype.Ptr(WSAPROTOCOL_INFOW)], + rffi.INT, save_err=SAVE_ERR) + + WSASocketW = external('WSASocketW', + [rffi.INT, rffi.INT, rffi.INT, + lltype.Ptr(WSAPROTOCOL_INFOW), + rwin32.DWORD, rwin32.DWORD], + socketfd_type, save_err=SAVE_ERR) + tcp_keepalive = cConfig.tcp_keepalive WSAPROTOCOL_INFO = cConfig.WSAPROTOCOL_INFO From pypy.commits at gmail.com Wed Jun 12 07:50:54 2019 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 12 Jun 2019 04:50:54 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge default Message-ID: <5d00e71e.1c69fb81.b0c9d.bab0@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: py3.6 Changeset: r96798:60c42a75264d Date: 2019-06-12 13:49 +0200 http://bitbucket.org/pypy/pypy/changeset/60c42a75264d/ Log: merge default diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -1011,7 +1011,7 @@ CConfig.WSAPROTOCOL_INFO = platform.Struct( 'WSAPROTOCOL_INFOA', []) # Struct is just passed between functions - + CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( 'FROM_PROTOCOL_INFO') @@ -1038,21 +1038,21 @@ ('keepaliveinterval', rffi.ULONG)]) CConfig.GUID = platform.Struct( - 'struct _GUID', - [('Data1', rffi.UINT), + 'struct _GUID', + [('Data1', rffi.UINT), ('Data2', rffi.UINT), ('Data3', rffi.UINT), ('Data4', rffi.CFixedArray(rffi.UCHAR, 8)) ]) - + CConfig.WSAPROTOCOLCHAIN = platform.Struct( 'struct _WSAPROTOCOLCHAIN', [('ChainLen', rffi.INT), ('ChainEntries', rffi.CFixedArray(rffi.UINT, 7))]) - + WSAPROTOCOLCHAIN = CConfig.WSAPROTOCOLCHAIN - GUID = CConfig.GUID - + GUID = CConfig.GUID + CConfig.WSAPROTOCOL_INFOW = platform.Struct( 'struct _WSAPROTOCOL_INFOW', [('dwServiceFlags1', rffi.UINT), @@ -1391,7 +1391,7 @@ lltype.Ptr(WSAPROTOCOL_INFOW), rwin32.DWORD, rwin32.DWORD], socketfd_type, save_err=SAVE_ERR) - + tcp_keepalive = cConfig.tcp_keepalive WSAPROTOCOL_INFO = cConfig.WSAPROTOCOL_INFO From pypy.commits at gmail.com Wed Jun 12 08:21:46 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 12 Jun 2019 05:21:46 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: sync with default Message-ID: <5d00ee5a.1c69fb81.9b93b.079f@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96799:dfa5401cba82 Date: 2019-06-12 15:04 +0300 http://bitbucket.org/pypy/pypy/changeset/dfa5401cba82/ Log: sync with default diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -1036,7 +1036,7 @@ [('onoff', rffi.ULONG), ('keepalivetime', rffi.ULONG), ('keepaliveinterval', rffi.ULONG)]) - + CConfig.GUID = platform.Struct( 'struct _GUID', [('Data1', rffi.UINT), From pypy.commits at gmail.com Wed Jun 12 10:25:43 2019 From: pypy.commits at gmail.com (rlamy) Date: Wed, 12 Jun 2019 07:25:43 -0700 (PDT) Subject: [pypy-commit] pypy default: Update warning: perf is now called pyperf Message-ID: <5d010b67.1c69fb81.ab512.1336@mx.google.com> Author: Ronan Lamy Branch: Changeset: r96800:32dd16507042 Date: 2019-06-12 15:24 +0100 http://bitbucket.org/pypy/pypy/changeset/32dd16507042/ Log: Update warning: perf is now called pyperf diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py --- a/lib-python/2.7/timeit.py +++ b/lib-python/2.7/timeit.py @@ -308,10 +308,10 @@ return 0 setup = "\n".join(setup) or "pass" - print "WARNING: timeit is a very unreliable tool. use perf or something else for real measurements" + print "WARNING: timeit is a very unreliable tool. use pyperf or something else for real measurements" executable = os.path.basename(sys.executable) - print "%s -m pip install perf" % executable - print "%s -m perf timeit %s" % ( + print "%s -m pip install pyperf" % executable + print "%s -m pyperf timeit %s" % ( executable, " ".join([(arg if arg.startswith("-") else repr(arg)) for arg in origargs]), ) From pypy.commits at gmail.com Wed Jun 12 10:28:53 2019 From: pypy.commits at gmail.com (rlamy) Date: Wed, 12 Jun 2019 07:28:53 -0700 (PDT) Subject: [pypy-commit] pypy default: fix test as well Message-ID: <5d010c25.1c69fb81.80144.0e9c@mx.google.com> Author: Ronan Lamy Branch: Changeset: r96801:e53c20a6841a Date: 2019-06-12 15:28 +0100 http://bitbucket.org/pypy/pypy/changeset/e53c20a6841a/ Log: fix test as well diff --git a/lib-python/2.7/test/test_timeit.py b/lib-python/2.7/test/test_timeit.py --- a/lib-python/2.7/test/test_timeit.py +++ b/lib-python/2.7/test/test_timeit.py @@ -317,9 +317,9 @@ def test_main_recommends_perf(self): s = self.run_main(seconds_per_increment=2.0, switches=['-n35', '-s', 'print("CustomSetup")']) self.assertIn(dedent("""\ - WARNING: timeit is a very unreliable tool. use perf or something else for real measurements + WARNING: timeit is a very unreliable tool. use pyperf or something else for real measurements """), s) - self.assertIn("-m pip install perf", s) + self.assertIn("-m pip install pyperf", s) From pypy.commits at gmail.com Wed Jun 12 10:32:27 2019 From: pypy.commits at gmail.com (rlamy) Date: Wed, 12 Jun 2019 07:32:27 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Update warning: perf is now called pyperf Message-ID: <5d010cfb.1c69fb81.15962.6aa5@mx.google.com> Author: Ronan Lamy Branch: py3.6 Changeset: r96802:c52c6b7bcc14 Date: 2019-06-12 15:31 +0100 http://bitbucket.org/pypy/pypy/changeset/c52c6b7bcc14/ Log: Update warning: perf is now called pyperf diff --git a/lib-python/3/test/test_timeit.py b/lib-python/3/test/test_timeit.py --- a/lib-python/3/test/test_timeit.py +++ b/lib-python/3/test/test_timeit.py @@ -366,9 +366,9 @@ def test_main_recommends_perf(self): s = self.run_main(seconds_per_increment=2.0, switches=['-n35', '-s', 'print("CustomSetup")']) self.assertIn(dedent("""\ - WARNING: timeit is a very unreliable tool. use perf or something else for real measurements + WARNING: timeit is a very unreliable tool. use pyperf or something else for real measurements """), s) - self.assertIn("-m pip install perf", s) + self.assertIn("-m pip install pyperf", s) def autorange(self, callback=None): timer = FakeTimer(seconds_per_increment=0.001) diff --git a/lib-python/3/timeit.py b/lib-python/3/timeit.py --- a/lib-python/3/timeit.py +++ b/lib-python/3/timeit.py @@ -310,10 +310,10 @@ return 0 setup = "\n".join(setup) or "pass" - print("WARNING: timeit is a very unreliable tool. use perf or something else for real measurements") + print("WARNING: timeit is a very unreliable tool. use pyperf or something else for real measurements") executable = os.path.basename(sys.executable) - print("%s -m pip install perf" % executable) - print("%s -m perf timeit %s" % ( + print("%s -m pip install pyperf" % executable) + print("%s -m pyperf timeit %s" % ( executable, " ".join([(arg if arg.startswith("-") else repr(arg)) for arg in origargs]), )) From pypy.commits at gmail.com Wed Jun 12 11:58:35 2019 From: pypy.commits at gmail.com (arigo) Date: Wed, 12 Jun 2019 08:58:35 -0700 (PDT) Subject: [pypy-commit] cffi default: Issue #412 Message-ID: <5d01212b.1c69fb81.bf61d.0dea@mx.google.com> Author: Armin Rigo Branch: Changeset: r3279:e00484ba0031 Date: 2019-06-12 17:58 +0200 http://bitbucket.org/cffi/cffi/changeset/e00484ba0031/ Log: Issue #412 Test and fix for unnamed bitfields which are not ":0" bitfields. diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -5275,16 +5275,18 @@ prev_bitfield_free -= fbitsize; field_offset_bytes = boffset / 8 - ftype->ct_size; } - if (sflags & SF_GCC_BIG_ENDIAN) bitshift = 8 * ftype->ct_size - fbitsize - bitshift; - *previous = _add_field(interned_fields, fname, ftype, + if (PyText_GetSize(fname) > 0) { + + *previous = _add_field(interned_fields, fname, ftype, field_offset_bytes, bitshift, fbitsize, fflags); - if (*previous == NULL) - goto error; - previous = &(*previous)->cf_next; + if (*previous == NULL) + goto error; + previous = &(*previous)->cf_next; + } } } diff --git a/cffi/recompiler.py b/cffi/recompiler.py --- a/cffi/recompiler.py +++ b/cffi/recompiler.py @@ -855,8 +855,9 @@ try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double - prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " - "an integer */" % (fname, cname, fname)) + if fname != '': + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " + "an integer */" % (fname, cname, fname)) continue # only accept exactly the type declared, except that '[]' # is interpreted as a '*' and so will match any array length. diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py --- a/testing/cffi1/test_recompiler.py +++ b/testing/cffi1/test_recompiler.py @@ -2359,3 +2359,56 @@ assert q[0].a == p.a assert q[0].b == p.b assert q == p + +def test_unnamed_bitfield_1(): + ffi = FFI() + ffi.cdef("""struct A { char : 1; };""") + lib = verify(ffi, "test_unnamed_bitfield_1", """ + struct A { char : 1; }; + """) + p = ffi.new("struct A *") + assert ffi.sizeof(p[0]) == 1 + # Note: on gcc, the type name is ignored for anonymous bitfields + # and that's why the result is 1. On MSVC, the result is + # sizeof("char") which is also 1. + +def test_unnamed_bitfield_2(): + ffi = FFI() + ffi.cdef("""struct A { + short c : 1; short : 1; short d : 1; short : 1; };""") + lib = verify(ffi, "test_unnamed_bitfield_2", """ + struct A { + short c : 1; short : 1; short d : 1; short : 1; + }; + """) + p = ffi.new("struct A *") + assert ffi.sizeof(p[0]) == ffi.sizeof("short") + +def test_unnamed_bitfield_3(): + ffi = FFI() + ffi.cdef("""struct A { struct { char : 1; char : 1; } b; };""") + lib = verify(ffi, "test_unnamed_bitfield_3", """ + struct A { struct { char : 1; char : 1; } b; }; + """) + p = ffi.new("struct A *") + assert ffi.sizeof(p[0]) == 1 + # Note: on gcc, the type name is ignored for anonymous bitfields + # and that's why the result is 1. On MSVC, the result is + # sizeof("char") which is also 1. + +def test_unnamed_bitfield_4(): + ffi = FFI() + ffi.cdef("""struct A { struct { + unsigned c : 1; unsigned : 1; unsigned d : 1; unsigned : 1; } a; + }; + struct B { struct A a; };""") + lib = verify(ffi, "test_unnamed_bitfield_4", """ + struct A { struct { + unsigned c : 1; unsigned : 1; unsigned d : 1; unsigned : 1; } a; + }; + struct B { struct A a; }; + """) + b = ffi.new("struct B *") + a = ffi.new("struct A *") + assert ffi.sizeof(a[0]) == ffi.sizeof("unsigned") + assert ffi.sizeof(b[0]) == ffi.sizeof(a[0]) From pypy.commits at gmail.com Thu Jun 13 06:56:04 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 13 Jun 2019 03:56:04 -0700 (PDT) Subject: [pypy-commit] pypy default: stringify constant Message-ID: <5d022bc4.1c69fb81.a23fc.4a58@mx.google.com> Author: Matti Picus Branch: Changeset: r96803:446e137abfba Date: 2019-06-13 13:53 +0300 http://bitbucket.org/pypy/pypy/changeset/446e137abfba/ Log: stringify constant diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -86,7 +86,7 @@ arch = platform.machine() g['LDSHARED'] += ' -undefined dynamic_lookup' g['CC'] += ' -arch %s' % (arch,) - g['MACOSX_DEPLOYMENT_TARGET'] = 10.14 + g['MACOSX_DEPLOYMENT_TARGET'] = '10.14' global _config_vars _config_vars = g From pypy.commits at gmail.com Fri Jun 14 03:25:33 2019 From: pypy.commits at gmail.com (stevie_92) Date: Fri, 14 Jun 2019 00:25:33 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-gc-cycle: Fixed issue during forking with rrc cycle detection Message-ID: <5d034bed.1c69fb81.a4375.78d2@mx.google.com> Author: Stefan Beyer Branch: cpyext-gc-cycle Changeset: r96804:d4f961b90303 Date: 2019-06-14 09:24 +0200 http://bitbucket.org/pypy/pypy/changeset/d4f961b90303/ Log: Fixed issue during forking with rrc cycle detection Fixed issue with non-rc rrc objects diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -3151,6 +3151,7 @@ self.rrc_finalizer_type = finalizer_type self.rrc_clear_weakref_callback = clear_weakref_callback self.rrc_enabled = True + self.rrc_cycle_enabled = True self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT def check_no_more_rawrefcount_state(self): @@ -3164,6 +3165,12 @@ self.rrc_p_dict.foreach(check_value_is_null, None) self.rrc_p_dict_nurs.foreach(check_value_is_null, None) + def deactivate_rawrefcount_cycle(self): + self.rrc_cycle_enabled = False + + def activate_rawrefcount_cycle(self): + self.rrc_cycle_enabled = True + def rawrefcount_create_link_pypy(self, gcobj, pyobject): ll_assert(self.rrc_enabled, "rawrefcount.init not called") obj = llmemory.cast_ptr_to_adr(gcobj) @@ -3405,10 +3412,13 @@ _rrc_free._always_inline_ = True def rrc_major_collection_trace(self): - self._rrc_debug_check_consistency(print_label="begin-mark") + if not self.rrc_cycle_enabled: + self._rrc_debug_check_consistency(print_label="begin-mark") + # Only trace and mark rawrefcounted object if we are not doing # something special, like building gc.garbage. - if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: + if (self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT and + self.rrc_cycle_enabled): merged_old_list = False # check objects with finalizers from last collection cycle if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): @@ -3477,6 +3487,30 @@ self.objects_to_trace.append(obj) self.visit_all_objects() + def _rrc_major_trace_nongc(self, pyobject, ignore): + from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY + from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT + # + pyobj = self._pyobj(pyobject) + pygchdr = self.rrc_pyobj_as_gc(pyobj) + if pygchdr != lltype.nullptr(self.PYOBJ_GC_HDR): + if pygchdr.c_gc_refs != self.RAWREFCOUNT_REFS_UNTRACKED: + rc = 0 + else: + rc = pyobj.c_ob_refcnt + else: + rc = pyobj.c_ob_refcnt + + if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT or rc == 0: + pass # the corresponding object may die + else: + # force the corresponding object to be alive + debug_print("pyobj stays alive", pyobj, "rc", rc) + intobj = pyobj.c_ob_pypy_link + obj = llmemory.cast_int_to_adr(intobj) + self.objects_to_trace.append(obj) + self.visit_all_objects() + def rrc_major_collection_free(self): if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): @@ -3572,6 +3606,10 @@ self._rrc_pyobj_gc_refcnt_set(pygchdr, refcnt) pygchdr = pygchdr.c_gc_next + # For all non-gc pyobjects which have a refcount > 0, + # mark all reachable objects on the pypy side + self.rrc_p_list_old.foreach(self._rrc_major_trace_nongc, None) + # For every object in this set, if it is marked, add 1 as a real # refcount (p_list => pyobj stays alive if obj stays alive). self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None) diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py --- a/rpython/memory/gctransform/shadowstack.py +++ b/rpython/memory/gctransform/shadowstack.py @@ -187,6 +187,7 @@ def switch_shadow_stacks(new_tid): # we have the wrong shadowstack right now, but it should not matter + gcdata.gc.deactivate_rawrefcount_cycle() thread_stacks = gcdata.thread_stacks try: if thread_stacks is None: @@ -210,6 +211,7 @@ shadow_stack_pool.start_fresh_new_state() # done # + gcdata.gc.activate_rawrefcount_cycle() gcdata.active_tid = new_tid switch_shadow_stacks._dont_inline_ = True From pypy.commits at gmail.com Fri Jun 14 11:12:26 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 14 Jun 2019 08:12:26 -0700 (PDT) Subject: [pypy-commit] pypy default: unclear whether it is easier to fix strerror_unicode or remove it, fixing for now Message-ID: <5d03b95a.1c69fb81.df15.4440@mx.google.com> Author: Matti Picus Branch: Changeset: r96805:1380de78f3e9 Date: 2019-06-14 17:57 +0300 http://bitbucket.org/pypy/pypy/changeset/1380de78f3e9/ Log: unclear whether it is easier to fix strerror_unicode or remove it, fixing for now diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -1426,10 +1426,10 @@ return rwin32.FormatError(errno) def socket_strerror_unicode(errno): - return rwin32.FormatErrorW(errno)[0] + return rwin32.FormatErrorW(errno)[0].decode('utf-8') def gai_strerror_unicode(errno): - return rwin32.FormatErrorW(errno)[0] + return rwin32.FormatErrorW(errno)[0].decode('utf-8') def socket_strerror_utf8(errno): return rwin32.FormatErrorW(errno) From pypy.commits at gmail.com Fri Jun 14 11:12:28 2019 From: pypy.commits at gmail.com (mattip) Date: Fri, 14 Jun 2019 08:12:28 -0700 (PDT) Subject: [pypy-commit] pypy default: resync vmprof files Message-ID: <5d03b95c.1c69fb81.527a3.d5e5@mx.google.com> Author: Matti Picus Branch: Changeset: r96806:728610cf10c0 Date: 2019-06-14 18:11 +0300 http://bitbucket.org/pypy/pypy/changeset/728610cf10c0/ Log: resync vmprof files diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -82,12 +82,6 @@ static PY_STACK_FRAME_T * _write_python_stack_entry(PY_STACK_FRAME_T * frame, void ** result, int * depth, int max_depth) { - int len; - int addr; - int j; - uint64_t line; - char *lnotab; - #ifndef RPYTHON_VMPROF // pypy does not support line profiling if (vmp_profiles_python_lines()) { // In the line profiling mode we save a line number for every frame. @@ -99,27 +93,8 @@ // NOTE: the profiling overhead can be reduced by storing co_lnotab in the dump and // moving this computation to the reader instead of doing it here. - lnotab = PyStr_AS_STRING(frame->f_code->co_lnotab); - - if (lnotab != NULL) { - line = (uint64_t)frame->f_lineno; - addr = 0; - - len = (int)PyStr_GET_SIZE(frame->f_code->co_lnotab); - - for (j = 0; j < len; j += 2) { - addr += lnotab[j]; - if (addr > frame->f_lasti) { - break; - } - line += lnotab[j+1]; - } - result[*depth] = (void*) line; - *depth = *depth + 1; - } else { - result[*depth] = (void*) 0; - *depth = *depth + 1; - } + result[*depth] = (void*) (int64_t) PyFrame_GetLineNumber(frame); + *depth = *depth + 1; } result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame)); *depth = *depth + 1; diff --git a/rpython/rlib/rvmprof/test/test_file.py b/rpython/rlib/rvmprof/test/test_file.py --- a/rpython/rlib/rvmprof/test/test_file.py +++ b/rpython/rlib/rvmprof/test/test_file.py @@ -37,7 +37,7 @@ # if no_matches: print - print 'The following file dit NOT match' + print 'The following file did NOT match' for f in no_matches: print ' ', f.relto(RVMPROF) raise AssertionError("some files were updated on github, " From pypy.commits at gmail.com Sat Jun 15 02:57:35 2019 From: pypy.commits at gmail.com (arigo) Date: Fri, 14 Jun 2019 23:57:35 -0700 (PDT) Subject: [pypy-commit] cffi default: add #412 in whatsnew Message-ID: <5d0496df.1c69fb81.47725.8dbf@mx.google.com> Author: Armin Rigo Branch: Changeset: r3280:8cca638c6631 Date: 2019-06-15 08:57 +0200 http://bitbucket.org/cffi/cffi/changeset/8cca638c6631/ Log: add #412 in whatsnew diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst --- a/doc/source/whatsnew.rst +++ b/doc/source/whatsnew.rst @@ -11,6 +11,8 @@ of ``p[0].field``. Be careful that no bounds checking is performed, so ``p[n]`` might access data out of bounds. +* fix for structs containing unnamed bitfields like ``int : 1;``. + v1.12.3 ======= From pypy.commits at gmail.com Sat Jun 15 03:17:58 2019 From: pypy.commits at gmail.com (arigo) Date: Sat, 15 Jun 2019 00:17:58 -0700 (PDT) Subject: [pypy-commit] pypy default: Import test from cffi/e00484ba0031. Fix it. Message-ID: <5d049ba6.1c69fb81.6464b.e6a1@mx.google.com> Author: Armin Rigo Branch: Changeset: r96807:328b9f323d42 Date: 2019-06-12 18:09 +0200 http://bitbucket.org/pypy/pypy/changeset/328b9f323d42/ Log: Import test from cffi/e00484ba0031. Fix it. diff --git a/extra_tests/cffi_tests/cffi1/test_recompiler.py b/extra_tests/cffi_tests/cffi1/test_recompiler.py --- a/extra_tests/cffi_tests/cffi1/test_recompiler.py +++ b/extra_tests/cffi_tests/cffi1/test_recompiler.py @@ -2360,3 +2360,56 @@ assert q[0].a == p.a assert q[0].b == p.b assert q == p + +def test_unnamed_bitfield_1(): + ffi = FFI() + ffi.cdef("""struct A { char : 1; };""") + lib = verify(ffi, "test_unnamed_bitfield_1", """ + struct A { char : 1; }; + """) + p = ffi.new("struct A *") + assert ffi.sizeof(p[0]) == 1 + # Note: on gcc, the type name is ignored for anonymous bitfields + # and that's why the result is 1. On MSVC, the result is + # sizeof("char") which is also 1. + +def test_unnamed_bitfield_2(): + ffi = FFI() + ffi.cdef("""struct A { + short c : 1; short : 1; short d : 1; short : 1; };""") + lib = verify(ffi, "test_unnamed_bitfield_2", """ + struct A { + short c : 1; short : 1; short d : 1; short : 1; + }; + """) + p = ffi.new("struct A *") + assert ffi.sizeof(p[0]) == ffi.sizeof("short") + +def test_unnamed_bitfield_3(): + ffi = FFI() + ffi.cdef("""struct A { struct { char : 1; char : 1; } b; };""") + lib = verify(ffi, "test_unnamed_bitfield_3", """ + struct A { struct { char : 1; char : 1; } b; }; + """) + p = ffi.new("struct A *") + assert ffi.sizeof(p[0]) == 1 + # Note: on gcc, the type name is ignored for anonymous bitfields + # and that's why the result is 1. On MSVC, the result is + # sizeof("char") which is also 1. + +def test_unnamed_bitfield_4(): + ffi = FFI() + ffi.cdef("""struct A { struct { + unsigned c : 1; unsigned : 1; unsigned d : 1; unsigned : 1; } a; + }; + struct B { struct A a; };""") + lib = verify(ffi, "test_unnamed_bitfield_4", """ + struct A { struct { + unsigned c : 1; unsigned : 1; unsigned d : 1; unsigned : 1; } a; + }; + struct B { struct A a; }; + """) + b = ffi.new("struct B *") + a = ffi.new("struct A *") + assert ffi.sizeof(a[0]) == ffi.sizeof("unsigned") + assert ffi.sizeof(b[0]) == ffi.sizeof(a[0]) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -855,8 +855,9 @@ try: if ftype.is_integer_type() or fbitsize >= 0: # accept all integers, but complain on float or double - prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " - "an integer */" % (fname, cname, fname)) + if fname != '': + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " + "an integer */" % (fname, cname, fname)) continue # only accept exactly the type declared, except that '[]' # is interpreted as a '*' and so will match any array length. diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -549,10 +549,11 @@ if sflags & SF_GCC_BIG_ENDIAN: bitshift = 8 * ftype.size - fbitsize- bitshift - fld = ctypestruct.W_CField(ftype, field_offset_bytes, - bitshift, fbitsize, fflags) - fields_list.append(fld) - fields_dict[fname] = fld + if fname != '': + fld = ctypestruct.W_CField(ftype, field_offset_bytes, + bitshift, fbitsize, fflags) + fields_list.append(fld) + fields_dict[fname] = fld if boffset > boffsetmax: boffsetmax = boffset From pypy.commits at gmail.com Sat Jun 15 03:18:00 2019 From: pypy.commits at gmail.com (arigo) Date: Sat, 15 Jun 2019 00:18:00 -0700 (PDT) Subject: [pypy-commit] pypy default: merge heads Message-ID: <5d049ba8.1c69fb81.20ee6.1133@mx.google.com> Author: Armin Rigo Branch: Changeset: r96808:bc7aeb4d5987 Date: 2019-06-15 09:07 +0200 http://bitbucket.org/pypy/pypy/changeset/bc7aeb4d5987/ Log: merge heads diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py --- a/lib-python/2.7/distutils/sysconfig_pypy.py +++ b/lib-python/2.7/distutils/sysconfig_pypy.py @@ -86,7 +86,7 @@ arch = platform.machine() g['LDSHARED'] += ' -undefined dynamic_lookup' g['CC'] += ' -arch %s' % (arch,) - g['MACOSX_DEPLOYMENT_TARGET'] = 10.14 + g['MACOSX_DEPLOYMENT_TARGET'] = '10.14' global _config_vars _config_vars = g diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -1426,10 +1426,10 @@ return rwin32.FormatError(errno) def socket_strerror_unicode(errno): - return rwin32.FormatErrorW(errno)[0] + return rwin32.FormatErrorW(errno)[0].decode('utf-8') def gai_strerror_unicode(errno): - return rwin32.FormatErrorW(errno)[0] + return rwin32.FormatErrorW(errno)[0].decode('utf-8') def socket_strerror_utf8(errno): return rwin32.FormatErrorW(errno) diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -82,12 +82,6 @@ static PY_STACK_FRAME_T * _write_python_stack_entry(PY_STACK_FRAME_T * frame, void ** result, int * depth, int max_depth) { - int len; - int addr; - int j; - uint64_t line; - char *lnotab; - #ifndef RPYTHON_VMPROF // pypy does not support line profiling if (vmp_profiles_python_lines()) { // In the line profiling mode we save a line number for every frame. @@ -99,27 +93,8 @@ // NOTE: the profiling overhead can be reduced by storing co_lnotab in the dump and // moving this computation to the reader instead of doing it here. - lnotab = PyStr_AS_STRING(frame->f_code->co_lnotab); - - if (lnotab != NULL) { - line = (uint64_t)frame->f_lineno; - addr = 0; - - len = (int)PyStr_GET_SIZE(frame->f_code->co_lnotab); - - for (j = 0; j < len; j += 2) { - addr += lnotab[j]; - if (addr > frame->f_lasti) { - break; - } - line += lnotab[j+1]; - } - result[*depth] = (void*) line; - *depth = *depth + 1; - } else { - result[*depth] = (void*) 0; - *depth = *depth + 1; - } + result[*depth] = (void*) (int64_t) PyFrame_GetLineNumber(frame); + *depth = *depth + 1; } result[*depth] = (void*)CODE_ADDR_TO_UID(FRAME_CODE(frame)); *depth = *depth + 1; diff --git a/rpython/rlib/rvmprof/test/test_file.py b/rpython/rlib/rvmprof/test/test_file.py --- a/rpython/rlib/rvmprof/test/test_file.py +++ b/rpython/rlib/rvmprof/test/test_file.py @@ -37,7 +37,7 @@ # if no_matches: print - print 'The following file dit NOT match' + print 'The following file did NOT match' for f in no_matches: print ' ', f.relto(RVMPROF) raise AssertionError("some files were updated on github, " From pypy.commits at gmail.com Sun Jun 16 01:12:57 2019 From: pypy.commits at gmail.com (mattip) Date: Sat, 15 Jun 2019 22:12:57 -0700 (PDT) Subject: [pypy-commit] pypy default: add gc.collect to fix test on bencher4 Message-ID: <5d05cfd9.1c69fb81.5e139.e690@mx.google.com> Author: Matti Picus Branch: Changeset: r96809:53fb62020937 Date: 2019-06-16 08:11 +0300 http://bitbucket.org/pypy/pypy/changeset/53fb62020937/ Log: add gc.collect to fix test on bencher4 diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -5,6 +5,7 @@ def test_local_1(self): import thread + import gc from thread import _local as tlsobject freed = [] class X: @@ -34,8 +35,9 @@ thread.start_new_thread(f, (i,)) self.waitfor(lambda: len(ok) == 20, delay=3) assert ok == 20*[True] # see stdout/stderr for failures in the threads + gc.collect(); gc.collect(); gc.collect() - self.waitfor(lambda: len(freed) >= 40) + self.waitfor(lambda: len(freed) >= 40, delay=20) assert len(freed) == 40 # in theory, all X objects should have been freed by now. Note that # Python's own thread._local objects suffer from the very same "bug" that From pypy.commits at gmail.com Mon Jun 17 04:27:53 2019 From: pypy.commits at gmail.com (arigo) Date: Mon, 17 Jun 2019 01:27:53 -0700 (PDT) Subject: [pypy-commit] pypy default: fix translation on arm Message-ID: <5d074f09.1c69fb81.c7e12.4778@mx.google.com> Author: Armin Rigo Branch: Changeset: r96810:c82849260828 Date: 2019-06-17 10:27 +0200 http://bitbucket.org/pypy/pypy/changeset/c82849260828/ Log: fix translation on arm diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -835,8 +835,9 @@ assert 0 def emit_op_load_effective_address(self, op, arglocs, regalloc, fcond): - self._gen_address(arglocs[4], arglocs[0], arglocs[1], arglocs[3].value, - arglocs[2].value) + static_ofs = op.getarg(2).getint() + scale = op.getarg(3).getint() + self._gen_address(arglocs[2], arglocs[0], arglocs[1], scale, static_ofs) return fcond # result = base_loc + (scaled_loc << scale) + static_offset diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -902,7 +902,7 @@ arg0 = self.make_sure_var_in_reg(args[0], args) arg1 = self.make_sure_var_in_reg(args[1], args) res = self.force_allocate_reg(op) - return [arg0, arg1, args[2], args[3], res] + return [arg0, arg1, res] def prepare_op_call_malloc_nursery(self, op, fcond): size_box = op.getarg(0) From pypy.commits at gmail.com Mon Jun 17 09:12:30 2019 From: pypy.commits at gmail.com (fijal) Date: Mon, 17 Jun 2019 06:12:30 -0700 (PDT) Subject: [pypy-commit] pypy arm64: start fighting with write barriers Message-ID: <5d0791be.1c69fb81.6a487.45e1@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96811:09bce457dc4b Date: 2019-06-17 13:11 +0000 http://bitbucket.org/pypy/pypy/changeset/09bce457dc4b/ Log: start fighting with write barriers diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -30,6 +30,7 @@ def __init__(self, cpu, translate_support_code=False): ResOpAssembler.__init__(self, cpu, translate_support_code) self.failure_recovery_code = [0, 0, 0, 0] + self.wb_slowpath = [0, 0, 0, 0, 0] def assemble_loop(self, jd_id, unique_id, logger, loopname, inputargs, operations, looptoken, log): @@ -318,7 +319,76 @@ self.mc.B(self.propagate_exception_path) def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): - pass # XXX + descr = self.cpu.gc_ll_descr.write_barrier_descr + if descr is None: + return + if not withcards: + func = descr.get_write_barrier_fn(self.cpu) + else: + if descr.jit_wb_cards_set == 0: + return + func = descr.get_write_barrier_from_array_fn(self.cpu) + if func == 0: + return + # + # This builds a helper function called from the slow path of + # write barriers. It must save all registers, and optionally + # all vfp registers. It takes a single argument which is in x0. + # It must keep stack alignment accordingly. + mc = InstrBuilder() + # + exc0 = exc1 = None + mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD) + mc.STR_ri(r.ip0.value, r.sp.value, WORD) + mc.STR_ri(r.lr.value, r.sp.value, 0) + if not for_frame: + self._push_all_regs_to_jitframe(mc, [], withfloats, callee_only=True) + else: + # NOTE: don't save registers on the jitframe here! It might + # override already-saved values that will be restored + # later... + # + # we're possibly called from the slowpath of malloc + # save the caller saved registers + # assuming we do not collect here + exc0, exc1 = r.r4, r.r5 + XXX + mc.PUSH([gpr.value for gpr in r.caller_resp] + [exc0.value, exc1.value]) + mc.VPUSH([vfpr.value for vfpr in r.caller_vfp_resp]) + + self._store_and_reset_exception(mc, exc0, exc1) + mc.BL(func) + # + if not for_frame: + self._pop_all_regs_from_jitframe(mc, [], withfloats, callee_only=True) + else: + XXX + self._restore_exception(mc, exc0, exc1) + mc.VPOP([vfpr.value for vfpr in r.caller_vfp_resp]) + assert exc0 is not None + assert exc1 is not None + mc.POP([gpr.value for gpr in r.caller_resp] + + [exc0.value, exc1.value]) + # + if withcards: + # A final TEST8 before the RET, for the caller. Careful to + # not follow this instruction with another one that changes + # the status of the CPU flags! + YYY + mc.LDRB_ri(r.ip.value, r.r0.value, + imm=descr.jit_wb_if_flag_byteofs) + mc.TST_ri(r.ip.value, imm=0x80) + # + mc.LDR_ri(r.ip0.value, r.sp.value, WORD) + mc.LDR_ri(r.ip1.value, r.sp.value, 0) + mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD) + mc.RET_r(r.ip1.value) + # + rawstart = mc.materialize(self.cpu, []) + if for_frame: + self.wb_slowpath[4] = rawstart + else: + self.wb_slowpath[withcards + 2 * withfloats] = rawstart def build_frame_realloc_slowpath(self): # this code should do the following steps diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -256,6 +256,11 @@ self.write32((base << 21) | (0b11111 << 16) | (cond << 12) | (1 << 10) | (0b11111 << 5) | rd) + def TST_rr_shift(self, rn, rm, shift): + assert 0 <= shift <= 64 + base = 0b11101010000 + self.write32((base << 21) | (rm << 16) | (shift << 10) | (rn << 5) | 0b11111) + def NOP(self): self.write32(0b11010101000000110010000000011111) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -1,10 +1,12 @@ +from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp.history import (AbstractFailDescr, ConstInt, INT, FLOAT, REF) from rpython.jit.backend.aarch64 import registers as r +from rpython.jit.backend.aarch64.codebuilder import OverwritingBuilder from rpython.jit.backend.aarch64.callbuilder import Aarch64CallBuilder from rpython.jit.backend.arm import conditions as c -from rpython.jit.backend.aarch64.arch import JITFRAME_FIXED_SIZE +from rpython.jit.backend.aarch64.arch import JITFRAME_FIXED_SIZE, WORD from rpython.jit.backend.aarch64.locations import imm from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler from rpython.jit.backend.llsupport.gcmap import allocate_gcmap @@ -493,6 +495,128 @@ def emit_op_restore_exception(self, op, arglocs): self._restore_exception(self.mc, arglocs[1], arglocs[0]) + def emit_op_cond_call_gc_wb(self, op, arglocs): + self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs) + + def _write_barrier_fastpath(self, mc, descr, arglocs, array=False, is_frame=False): + # Write code equivalent to write_barrier() in the GC: it checks + # a flag in the object at arglocs[0], and if set, it calls a + # helper piece of assembler. The latter saves registers as needed + # and call the function remember_young_pointer() from the GC. + if we_are_translated(): + cls = self.cpu.gc_ll_descr.has_write_barrier_class() + assert cls is not None and isinstance(descr, cls) + # + card_marking = False + mask = descr.jit_wb_if_flag_singlebyte + if array and descr.jit_wb_cards_set != 0: + # assumptions the rest of the function depends on: + assert (descr.jit_wb_cards_set_byteofs == + descr.jit_wb_if_flag_byteofs) + assert descr.jit_wb_cards_set_singlebyte == -0x80 + card_marking = True + mask = descr.jit_wb_if_flag_singlebyte | -0x80 + # + loc_base = arglocs[0] + if is_frame: + assert loc_base is r.fp + mc.LDRB_ri(r.ip0.value, loc_base.value, descr.jit_wb_if_flag_byteofs) + mask &= 0xFF + mc.MOVZ_r_u16(r.ip1.value, mask, 0) + mc.TST_rr_shift(r.ip0.value, r.ip1.value, 0) + jz_location = mc.currpos() + mc.BRK() + + # for cond_call_gc_wb_array, also add another fast path: + # if GCFLAG_CARDS_SET, then we can just set one bit and be done + if card_marking: + XXX + # GCFLAG_CARDS_SET is in this byte at 0x80 + mc.TST_ri(r.ip.value, imm=0x80) + + js_location = mc.currpos() + mc.BKPT() + else: + js_location = 0 + + # Write only a CALL to the helper prepared in advance, passing it as + # argument the address of the structure we are writing into + # (the first argument to COND_CALL_GC_WB). + helper_num = card_marking + if is_frame: + helper_num = 4 + elif self._regalloc is not None and self._regalloc.vfprm.reg_bindings: + helper_num += 2 + if self.wb_slowpath[helper_num] == 0: # tests only + assert not we_are_translated() + self.cpu.gc_ll_descr.write_barrier_descr = descr + self._build_wb_slowpath(card_marking, + bool(self._regalloc.vfprm.reg_bindings)) + assert self.wb_slowpath[helper_num] != 0 + # + if loc_base is not r.x0: + # push two registers to keep stack aligned + mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD) + mc.STR_ri(r.x0.value, r.sp.value, WORD) + mc.STR_ri(loc_base.value, r.sp.value, 0) + mc.MOV_rr(r.x0.value, loc_base.value) + if is_frame: + assert loc_base is r.fp + mc.BL(self.wb_slowpath[helper_num]) + if loc_base is not r.x0: + mc.LDR_ri(r.x0.value, r.sp.value, WORD) + mc.LDR_ri(loc_base.value, r.sp.value, 0) + mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD) + + if card_marking: + # The helper ends again with a check of the flag in the object. So + # here, we can simply write again a conditional jump, which will be + # taken if GCFLAG_CARDS_SET is still not set. + jns_location = mc.currpos() + mc.BKPT() + # + # patch the JS above + offset = mc.currpos() + pmc = OverwritingBuilder(mc, js_location, WORD) + pmc.B_offs(offset, c.NE) # We want to jump if the z flag isn't set + # + # case GCFLAG_CARDS_SET: emit a few instructions to do + # directly the card flag setting + loc_index = arglocs[1] + assert loc_index.is_core_reg() + # must save the register loc_index before it is mutated + mc.PUSH([loc_index.value]) + tmp1 = loc_index + tmp2 = arglocs[-1] # the last item is a preallocated tmp + # lr = byteofs + s = 3 + descr.jit_wb_card_page_shift + mc.MVN_rr(r.lr.value, loc_index.value, + imm=s, shifttype=shift.LSR) + + # tmp1 = byte_index + mc.MOV_ri(r.ip.value, imm=7) + mc.AND_rr(tmp1.value, r.ip.value, loc_index.value, + imm=descr.jit_wb_card_page_shift, shifttype=shift.LSR) + + # set the bit + mc.MOV_ri(tmp2.value, imm=1) + mc.LDRB_rr(r.ip.value, loc_base.value, r.lr.value) + mc.ORR_rr_sr(r.ip.value, r.ip.value, tmp2.value, + tmp1.value, shifttype=shift.LSL) + mc.STRB_rr(r.ip.value, loc_base.value, r.lr.value) + # done + mc.POP([loc_index.value]) + # + # + # patch the JNS above + offset = mc.currpos() + pmc = OverwritingBuilder(mc, jns_location, WORD) + pmc.B_offs(offset, c.EQ) # We want to jump if the z flag is set + + offset = mc.currpos() - jz_location + pmc = OverwritingBuilder(mc, jz_location, WORD) + pmc.B_ofs_cond(offset, c.EQ) + # ----------------------------- call ------------------------------ def _genop_call(self, op, arglocs): diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -800,6 +800,15 @@ src_locations2, dst_locations2, vfptmploc) return [] + def prepare_op_cond_call_gc_wb(self, op): + # we force all arguments in a reg because it will be needed anyway by + # the following gc_store. It avoids loading it twice from the memory. + N = op.numargs() + args = op.getarglist() + arglocs = [self.make_sure_var_in_reg(op.getarg(i), args) + for i in range(N)] + return arglocs + def force_allocate_reg(self, var, forbidden_vars=[], selected_reg=None): if var.type == FLOAT: return self.vfprm.force_allocate_reg(var, forbidden_vars, From pypy.commits at gmail.com Mon Jun 17 10:09:32 2019 From: pypy.commits at gmail.com (fijal) Date: Mon, 17 Jun 2019 07:09:32 -0700 (PDT) Subject: [pypy-commit] pypy arm64: work more on write barrier Message-ID: <5d079f1c.1c69fb81.36d3.7cc5@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96812:7a1bbf798839 Date: 2019-06-17 14:08 +0000 http://bitbucket.org/pypy/pypy/changeset/7a1bbf798839/ Log: work more on write barrier diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -374,10 +374,9 @@ # A final TEST8 before the RET, for the caller. Careful to # not follow this instruction with another one that changes # the status of the CPU flags! - YYY - mc.LDRB_ri(r.ip.value, r.r0.value, - imm=descr.jit_wb_if_flag_byteofs) - mc.TST_ri(r.ip.value, imm=0x80) + mc.LDRB_ri(r.ip0.value, r.x0.value, descr.jit_wb_if_flag_byteofs) + mc.MOVZ_r_u16(r.ip1.value, 0x80, 0) + mc.TST_rr_shift(r.ip0.value, r.ip1.value, 0) # mc.LDR_ri(r.ip0.value, r.sp.value, WORD) mc.LDR_ri(r.ip1.value, r.sp.value, 0) diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -180,6 +180,11 @@ base = 0b10001010000 self.write32((base << 21) | (rm << 16) | (rn << 5) | rd) + def AND_rr_shift(self, rd, rn, rm, shift, shifttype=0): + base = 0b10001010 + assert 0 <= shift < 64 + self.write32((base << 24) | (shifttype << 22) | (rm << 16) | (shift << 10) | (rn << 5) | rd) + def AND_ri(self, rd, rn, immed): assert immed == 0xFF # just one value for now, don't feel like # understanding IMMR/IMMS quite yet @@ -225,10 +230,16 @@ base = 0b11001010000 self.write32((base << 21) | (rm << 16) | (rn << 5) | rd) - def MVN_rr(self, rd, rm): + def MVN_rr(self, rd, rm): # defaults to xzr base = 0b10101010001 self.write32((base << 21) | (rm << 16) | (0b11111 << 5)| rd) + def MVN_rr_shift(self, rd, rm, shift=0, shifttype=0): # defaults to LSL + base = 0b10101010 + assert 0 <= shift < 64 + self.write32((base << 24) | (shifttype << 22) | (1 << 21) | + (rm << 16) | (shift << 10) | (0b11111 << 5) | rd) + def SMULL_rr(self, rd, rn, rm): base = 0b10011011001 self.write32((base << 21) | (rm << 16) | (0b11111 << 10) | (rn << 5) | rd) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -5,7 +5,7 @@ from rpython.jit.backend.aarch64 import registers as r from rpython.jit.backend.aarch64.codebuilder import OverwritingBuilder from rpython.jit.backend.aarch64.callbuilder import Aarch64CallBuilder -from rpython.jit.backend.arm import conditions as c +from rpython.jit.backend.arm import conditions as c, shift from rpython.jit.backend.aarch64.arch import JITFRAME_FIXED_SIZE, WORD from rpython.jit.backend.aarch64.locations import imm from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler @@ -498,6 +498,10 @@ def emit_op_cond_call_gc_wb(self, op, arglocs): self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs) + def emit_op_cond_call_gc_wb_array(self, op, arglocs): + self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs, + array=True) + def _write_barrier_fastpath(self, mc, descr, arglocs, array=False, is_frame=False): # Write code equivalent to write_barrier() in the GC: it checks # a flag in the object at arglocs[0], and if set, it calls a @@ -530,12 +534,12 @@ # for cond_call_gc_wb_array, also add another fast path: # if GCFLAG_CARDS_SET, then we can just set one bit and be done if card_marking: - XXX + mc.MOVZ_r_u16(r.ip1.value, 0x80, 0) # GCFLAG_CARDS_SET is in this byte at 0x80 - mc.TST_ri(r.ip.value, imm=0x80) + mc.TST_rr_shift(r.ip0.value, r.ip1.value, 0) js_location = mc.currpos() - mc.BKPT() + mc.BRK() else: js_location = 0 @@ -573,45 +577,40 @@ # here, we can simply write again a conditional jump, which will be # taken if GCFLAG_CARDS_SET is still not set. jns_location = mc.currpos() - mc.BKPT() + mc.BRK() # # patch the JS above - offset = mc.currpos() + offset = mc.currpos() - js_location pmc = OverwritingBuilder(mc, js_location, WORD) - pmc.B_offs(offset, c.NE) # We want to jump if the z flag isn't set + pmc.B_ofs_cond(offset, c.NE) # We want to jump if the z flag isn't set # # case GCFLAG_CARDS_SET: emit a few instructions to do # directly the card flag setting loc_index = arglocs[1] assert loc_index.is_core_reg() - # must save the register loc_index before it is mutated - mc.PUSH([loc_index.value]) - tmp1 = loc_index + tmp1 = r.ip1 tmp2 = arglocs[-1] # the last item is a preallocated tmp # lr = byteofs s = 3 + descr.jit_wb_card_page_shift - mc.MVN_rr(r.lr.value, loc_index.value, - imm=s, shifttype=shift.LSR) + mc.MVN_rr_shift(r.lr.value, loc_index.value, s, shifttype=shift.LSR) # tmp1 = byte_index - mc.MOV_ri(r.ip.value, imm=7) - mc.AND_rr(tmp1.value, r.ip.value, loc_index.value, - imm=descr.jit_wb_card_page_shift, shifttype=shift.LSR) + mc.MOVZ_r_u16(r.ip0.value, 7, 0) + mc.AND_rr_shift(tmp1.value, r.ip0.value, loc_index.value, + descr.jit_wb_card_page_shift, shifttype=shift.LSR) # set the bit - mc.MOV_ri(tmp2.value, imm=1) - mc.LDRB_rr(r.ip.value, loc_base.value, r.lr.value) - mc.ORR_rr_sr(r.ip.value, r.ip.value, tmp2.value, - tmp1.value, shifttype=shift.LSL) - mc.STRB_rr(r.ip.value, loc_base.value, r.lr.value) + mc.MOVZ_r_u16(tmp2.value, 1, 0) + mc.LDRB_rr(r.ip0.value, loc_base.value, r.lr.value) + mc.LSL_rr(tmp2.value, tmp2.value, tmp1.value) + mc.ORR_rr(r.ip0.value, r.ip0.value, tmp2.value) + mc.STR_size_rr(0, r.ip0.value, loc_base.value, r.lr.value) # done - mc.POP([loc_index.value]) - # # # patch the JNS above - offset = mc.currpos() + offset = mc.currpos() - jns_location pmc = OverwritingBuilder(mc, jns_location, WORD) - pmc.B_offs(offset, c.EQ) # We want to jump if the z flag is set + pmc.B_ofs_cond(offset, c.EQ) # We want to jump if the z flag is set offset = mc.currpos() - jz_location pmc = OverwritingBuilder(mc, jz_location, WORD) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -809,6 +809,8 @@ for i in range(N)] return arglocs + prepare_op_cond_call_gc_wb_array = prepare_op_cond_call_gc_wb + def force_allocate_reg(self, var, forbidden_vars=[], selected_reg=None): if var.type == FLOAT: return self.vfprm.force_allocate_reg(var, forbidden_vars, From pypy.commits at gmail.com Mon Jun 17 14:17:16 2019 From: pypy.commits at gmail.com (rlamy) Date: Mon, 17 Jun 2019 11:17:16 -0700 (PDT) Subject: [pypy-commit] pypy default: Cleanup unicode_to_decimal_w() to make it more similar to pypy3 Message-ID: <5d07d92c.1c69fb81.081d.67ba@mx.google.com> Author: Ronan Lamy Branch: Changeset: r96814:cde3d214c398 Date: 2019-06-17 18:26 +0100 http://bitbucket.org/pypy/pypy/changeset/cde3d214c398/ Log: Cleanup unicode_to_decimal_w() to make it more similar to pypy3 diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -49,7 +49,6 @@ # special-case in Python 2, which is exactly what we want here assert length == len(utf8str.decode('utf-8')) - @staticmethod def from_utf8builder(builder): return W_UnicodeObject( @@ -1097,11 +1096,11 @@ if rutf8.has_surrogates(utf8): utf8 = rutf8.reencode_utf8_with_surrogates(utf8) return space.newbytes(utf8) - return encode(space, w_obj, encoding, errors) + return encode(space, w_obj, encoding, errors) def decode_object(space, w_obj, encoding, errors): - from pypy.module._codecs.interp_codecs import lookup_codec, decode + from pypy.module._codecs.interp_codecs import lookup_codec, decode if errors is None or errors == 'strict': # fast paths if encoding is None: @@ -1111,7 +1110,7 @@ unicodehelper.check_ascii_or_raise(space, s) return space.newutf8(s, len(s)) if encoding == 'utf-8' or encoding == 'utf8': - if (space.isinstance_w(w_obj, space.w_unicode) or + if (space.isinstance_w(w_obj, space.w_unicode) or space.isinstance_w(w_obj, space.w_bytes)): s = space.utf8_w(w_obj) else: @@ -1720,34 +1719,28 @@ def unicode_to_decimal_w(space, w_unistr): if not isinstance(w_unistr, W_UnicodeObject): raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr) - unistr = w_unistr._utf8 - result = ['\0'] * w_unistr._length - digits = ['0', '1', '2', '3', '4', - '5', '6', '7', '8', '9'] - res_pos = 0 - iter = rutf8.Utf8StringIterator(unistr) - for uchr in iter: + utf8 = w_unistr._utf8 + result = StringBuilder(w_unistr._len()) + it = rutf8.Utf8StringIterator(utf8) + for uchr in it: if W_UnicodeObject._isspace(uchr): - result[res_pos] = ' ' - res_pos += 1 + result.append(' ') continue - try: - result[res_pos] = digits[unicodedb.decimal(uchr)] - except KeyError: - if 0 < uchr < 256: - result[res_pos] = chr(uchr) - else: + if not (0 < uchr < 256): + try: + uchr = ord('0') + unicodedb.decimal(uchr) + except KeyError: w_encoding = space.newtext('decimal') - pos = iter.get_pos() + pos = it.get_pos() w_start = space.newint(pos) - w_end = space.newint(pos+1) + w_end = space.newint(pos + 1) w_reason = space.newtext('invalid decimal Unicode string') raise OperationError(space.w_UnicodeEncodeError, - space.newtuple([w_encoding, w_unistr, - w_start, w_end, - w_reason])) - res_pos += 1 - return ''.join(result) + space.newtuple([w_encoding, w_unistr, + w_start, w_end, + w_reason])) + result.append(chr(uchr)) + return result.build() _repr_function = rutf8.make_utf8_escape_function( From pypy.commits at gmail.com Mon Jun 17 15:04:35 2019 From: pypy.commits at gmail.com (rlamy) Date: Mon, 17 Jun 2019 12:04:35 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Optimise the common case for str to int conversion Message-ID: <5d07e443.1c69fb81.6449b.0ae4@mx.google.com> Author: Ronan Lamy Branch: py3.6 Changeset: r96815:102276149584 Date: 2019-06-17 20:03 +0100 http://bitbucket.org/pypy/pypy/changeset/102276149584/ Log: Optimise the common case for str to int conversion diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1882,6 +1882,14 @@ def unicode_to_decimal_w(space, w_unistr, allow_surrogates=False): if not isinstance(w_unistr, W_UnicodeObject): raise oefmt(space.w_TypeError, "expected unicode, got '%T'", w_unistr) + if w_unistr.is_ascii(): + # fast path + return w_unistr._utf8 + else: + return _unicode_to_decimal_w(space, w_unistr) + +def _unicode_to_decimal_w(space, w_unistr): + # slow path, in a separate function for the JIT's benefit utf8 = w_unistr._utf8 result = StringBuilder(w_unistr._len()) it = rutf8.Utf8StringIterator(utf8) From pypy.commits at gmail.com Tue Jun 18 03:02:55 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 18 Jun 2019 00:02:55 -0700 (PDT) Subject: [pypy-commit] pypy default: fix vmprof for 32 bit linux Message-ID: <5d088c9f.1c69fb81.92413.58e6@mx.google.com> Author: Matti Picus Branch: Changeset: r96816:c3622918429e Date: 2019-06-18 10:01 +0300 http://bitbucket.org/pypy/pypy/changeset/c3622918429e/ Log: fix vmprof for 32 bit linux diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c --- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c +++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c @@ -280,7 +280,7 @@ // this is possible because compiler align to 8 bytes. // if (func_addr != 0x0) { - depth = _write_native_stack((void*)(((uint64_t)func_addr) | 0x1), result, depth, max_depth); + depth = _write_native_stack((void*)(((intptr_t)func_addr) | 0x1), result, depth, max_depth); } } diff --git a/rpython/rlib/rvmprof/test/test_file.py b/rpython/rlib/rvmprof/test/test_file.py --- a/rpython/rlib/rvmprof/test/test_file.py +++ b/rpython/rlib/rvmprof/test/test_file.py @@ -11,10 +11,11 @@ def get_list_of_files(shared): files = list(shared.visit('*.[ch]')) # in PyPy we checkin the result of ./configure; as such, these files are - # not in github and can be skipped + # not in github or different and can be skipped files.remove(shared.join('libbacktrace', 'config-x86_32.h')) files.remove(shared.join('libbacktrace', 'config-x86_64.h')) files.remove(shared.join('libbacktrace', 'gstdint.h')) + files.remove(shared.join('libbacktrace', 'config.h')) return files def test_same_file(): From pypy.commits at gmail.com Tue Jun 18 04:56:10 2019 From: pypy.commits at gmail.com (arigo) Date: Tue, 18 Jun 2019 01:56:10 -0700 (PDT) Subject: [pypy-commit] pypy default: Remove @xfail on these two Windows tests. One of them passes right now (and so Message-ID: <5d08a72a.1c69fb81.92386.e8a9@mx.google.com> Author: Armin Rigo Branch: Changeset: r96817:6d85d83cc17b Date: 2019-06-18 10:55 +0200 http://bitbucket.org/pypy/pypy/changeset/6d85d83cc17b/ Log: Remove @xfail on these two Windows tests. One of them passes right now (and so it ends up as a failure). I don't see a reason for the other to be @xfail except because PyPy is buggy, and in this case it's better to see this (real) failure. diff --git a/lib-python/2.7/ctypes/test/test_loading.py b/lib-python/2.7/ctypes/test/test_loading.py --- a/lib-python/2.7/ctypes/test/test_loading.py +++ b/lib-python/2.7/ctypes/test/test_loading.py @@ -2,7 +2,7 @@ import sys, unittest import os from ctypes.util import find_library -from ctypes.test import is_resource_enabled, xfail +from ctypes.test import is_resource_enabled libc_name = None if os.name == "nt": @@ -80,7 +80,6 @@ self.assertRaises(AttributeError, dll.__getitem__, 1234) - @xfail @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_A(self): from _ctypes import LoadLibrary, FreeLibrary @@ -92,7 +91,6 @@ handle = LoadLibrary("advapi32") FreeLibrary(handle) - @xfail @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_B(self): # Since on winXP 64-bit advapi32 loads like described From pypy.commits at gmail.com Tue Jun 18 07:01:00 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 18 Jun 2019 04:01:00 -0700 (PDT) Subject: [pypy-commit] pypy arm64: start working on cond_call, we need float support for that Message-ID: <5d08c46c.1c69fb81.3e197.d32f@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96818:98896de049b1 Date: 2019-06-18 10:56 +0000 http://bitbucket.org/pypy/pypy/changeset/98896de049b1/ Log: start working on cond_call, we need float support for that diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -530,7 +530,26 @@ self.propagate_exception_path = rawstart def _build_cond_call_slowpath(self, supports_floats, callee_only): - pass + """ This builds a general call slowpath, for whatever call happens to + come. + """ + mc = InstrBuilder() + # + self._push_all_regs_to_jitframe(mc, [], self.cpu.supports_floats, callee_only) + ## args are in their respective positions + mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD) + mc.STR_ri(r.ip0.value, r.sp.value, WORD) + mc.STR_ri(r.lr.value, r.sp.value, 0) + mc.BL(r.ip1.value) + mc.MOV_rr(r.ip1.value, r.x0.value) # return comes back in ip1 + self._reload_frame_if_necessary(mc) + self._pop_all_regs_from_jitframe(mc, [], supports_floats, + callee_only) + # return + mc.LDR_ri(r.ip0.value, r.sp.value, 0) + mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD) + mc.RET_r(r.ip0.value) + return mc.materialize(self.cpu, []) def _build_stack_check_slowpath(self): self.stack_check_slowpath = 0 #XXX diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -1,5 +1,6 @@ from rpython.rlib.objectmodel import we_are_translated +from rpython.rtyper.lltypesystem import rffi, lltype from rpython.jit.metainterp.history import (AbstractFailDescr, ConstInt, INT, FLOAT, REF) from rpython.jit.backend.aarch64 import registers as r @@ -502,6 +503,49 @@ self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs, array=True) + def emit_op_cond_call(self, op, arglocs): + self.mc.gen_load_int(r.ip1.value, rffi.cast(lltype.Signed, + op.getarg(1).getint())) + + #if len(arglocs) == 2: + # res_loc = arglocs[1] # cond_call_value + #else: + # res_loc = None # cond_call + # see x86.regalloc for why we skip res_loc in the gcmap + res_loc = None + gcmap = self._regalloc.get_gcmap([res_loc]) + + jmp_adr = self.mc.currpos() + self.mc.BRK() # patched later: the conditional jump + # + self.push_gcmap(self.mc, gcmap) + # + callee_only = False + floats = False + if self._regalloc is not None: + for reg in self._regalloc.rm.reg_bindings.values(): + if reg not in self._regalloc.rm.save_around_call_regs: + break + else: + callee_only = True + if self._regalloc.vfprm.reg_bindings: + floats = True + cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only] + assert cond_call_adr + self.mc.BL(cond_call_adr) + # if this is a COND_CALL_VALUE, we need to move the result in place + # from its current location (which is, unusually, in r4: see + # cond_call_slowpath) + if res_loc is not None: + self.mc.MOV_rr(res_loc.value, r.ip1.value) + # + self.pop_gcmap(self.mc) + pmc = OverwritingBuilder(self.mc, jmp_adr, WORD) + pmc.B_ofs_cond(self.mc.currpos(), c.EQ) + # might be overridden again to skip over the following + # guard_no_exception too + self.previous_cond_call_jcond = jmp_adr, c.EQ + def _write_barrier_fastpath(self, mc, descr, arglocs, array=False, is_frame=False): # Write code equivalent to write_barrier() in the GC: it checks # a flag in the object at arglocs[0], and if set, it calls a diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -630,6 +630,42 @@ self._compute_hint_frame_locations_from_descr(descr) return [] + def prepare_op_cond_call(self, op): + assert 2 <= op.numargs() <= 4 + 2 + v = op.getarg(1) + assert isinstance(v, Const) + args_so_far = [] + for i in range(2, op.numargs()): + reg = r.argument_regs[i - 2] + arg = op.getarg(i) + self.make_sure_var_in_reg(arg, args_so_far, selected_reg=reg) + args_so_far.append(arg) + + if op.type == 'v': + # a plain COND_CALL. Calls the function when args[0] is + # true. Often used just after a comparison operation. + return [] + else: + XXX + # COND_CALL_VALUE_I/R. Calls the function when args[0] + # is equal to 0 or NULL. Returns the result from the + # function call if done, or args[0] if it was not 0/NULL. + # Implemented by forcing the result to live in the same + # register as args[0], and overwriting it if we really do + # the call. + + # Load the register for the result. Possibly reuse 'args[0]'. + # But the old value of args[0], if it survives, is first + # spilled away. We can't overwrite any of op.args[2:] here. + args = op.getarglist() + resloc = self.rm.force_result_in_reg(op, args[0], + forbidden_vars=args[2:]) + # Test the register for the result. + self.assembler.mc.CMP_ri(resloc.value, 0) + self.assembler.guard_success_cc = c.EQ + return [tmpreg, resloc] + + def prepare_op_finish(self, op): # the frame is in fp, but we have to point where in the frame is # the potential argument to FINISH From pypy.commits at gmail.com Tue Jun 18 08:12:33 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 18 Jun 2019 05:12:33 -0700 (PDT) Subject: [pypy-commit] pypy arm64: enough float support to start running tests Message-ID: <5d08d531.1c69fb81.6dcbe.33e4@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96819:fee018704c07 Date: 2019-06-18 12:11 +0000 http://bitbucket.org/pypy/pypy/changeset/fee018704c07/ Log: enough float support to start running tests diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -6,7 +6,7 @@ from rpython.jit.backend.aarch64.opassembler import ResOpAssembler from rpython.jit.backend.aarch64.regalloc import (Regalloc, check_imm_arg, operations as regalloc_operations, guard_operations, comp_operations, - CoreRegisterManager) + CoreRegisterManager, VFPRegisterManager) from rpython.jit.backend.aarch64 import registers as r from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.llsupport import jitframe @@ -248,9 +248,8 @@ # Push VFP regs regs = VFPRegisterManager.all_regs ofs = len(CoreRegisterManager.all_regs) * WORD - assert check_imm_arg(ofs+base_ofs) - mc.ADD_ri(r.ip.value, r.fp.value, imm=ofs+base_ofs) - mc.VSTM(r.ip.value, [vfpr.value for vfpr in regs]) + for reg in regs: + mc.STR_di(reg.value, r.fp.value, ofs + base_ofs + reg.value * WORD) def _pop_all_regs_from_jitframe(self, mc, ignored_regs, withfloats, callee_only=False): @@ -278,9 +277,8 @@ # Pop VFP regs regs = VFPRegisterManager.all_regs ofs = len(CoreRegisterManager.all_regs) * WORD - assert check_imm_arg(ofs+base_ofs) - mc.ADD_ri(r.ip.value, r.fp.value, imm=ofs+base_ofs) - mc.VLDM(r.ip.value, [vfpr.value for vfpr in regs]) + for reg in regs: + mc.LDR_di(reg.value, r.fp.value, ofs + base_ofs + reg.value * WORD) def _build_failure_recovery(self, exc, withfloats=False): mc = InstrBuilder() @@ -745,11 +743,6 @@ #self.saved_threadlocal_addr = 0 # at offset 0 from location 'sp' # ^^^XXX save it from register x1 into some place - if self.cpu.supports_floats: - XXX - self.mc.VPUSH([reg.value for reg in r.callee_saved_vfp_registers]) - self.saved_threadlocal_addr += ( - len(r.callee_saved_vfp_registers) * 2 * WORD) # set fp to point to the JITFRAME, passed in argument 'x0' self.mc.MOV_rr(r.fp.value, r.x0.value) @@ -916,13 +909,11 @@ mc = self.mc if gcrootmap and gcrootmap.is_shadow_stack: self.gen_footer_shadowstack(gcrootmap, mc) - if self.cpu.supports_floats: - XXX - # mc.VPOP([reg.value for reg in r.callee_saved_vfp_registers]) # pop all callee saved registers - stack_size = (len(r.callee_saved_registers) + 2) * WORD + stack_size = len(r.callee_saved_registers) * WORD + for i in range(0, len(r.callee_saved_registers), 2): mc.LDP_rri(r.callee_saved_registers[i].value, r.callee_saved_registers[i + 1].value, @@ -930,6 +921,7 @@ (i + 2) * WORD) mc.LDP_rr_postindex(r.fp.value, r.lr.value, r.sp.value, stack_size) + mc.RET_r(r.lr.value) def store_reg(self, mc, source, base, ofs=0, helper=None): diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -24,6 +24,12 @@ self.write32((base << 22) | ((offset >> 3) << 10) | (rn << 5) | rt) + def STR_di(self, rt, rn, offset): + base = 0b1111110100 + assert offset & 0x7 == 0 + assert 0 <= offset < 32768 + self.write32((base << 22) | ((offset >> 3) << 10) | (rn << 5) | rt) + def STP_rr_preindex(self, reg1, reg2, rn, offset): base = 0b1010100110 assert -512 <= offset < 512 @@ -110,6 +116,12 @@ assert immed & 0x7 == 0 self.write32((base << 22) | (immed >> 3 << 10) | (rn << 5) | rt) + def LDR_di(self, rt, rn, offset): + assert offset & 0x7 == 0 + assert 0 <= offset < 32768 + base = 0b1111110101 + self.write32((base << 22) | (offset >> 3 << 10) | (rn << 5) | rt) + def LDRB_ri(self, rt, rn, immed): base = 0b0011100101 assert 0 <= immed <= 1<<12 @@ -340,6 +352,8 @@ def get_max_size_of_gen_load_int(self): return 4 + # ------------------------------------------- + class OverwritingBuilder(AbstractAarch64Builder): def __init__(self, cb, start, size): diff --git a/rpython/jit/backend/aarch64/registers.py b/rpython/jit/backend/aarch64/registers.py --- a/rpython/jit/backend/aarch64/registers.py +++ b/rpython/jit/backend/aarch64/registers.py @@ -10,7 +10,7 @@ x21, x22, x23, x24, x25, x26, x27, x28, x29, x30] = registers vfpregisters = [VFPRegisterLocation(i) for i in range(32)] -all_vfp_regs = vfpregisters[:16] +all_vfp_regs = vfpregisters[:8] all_regs = registers[:16] #+ [x19, x20, x21, x22] lr = x30 @@ -22,6 +22,7 @@ ip0 = x16 callee_saved_registers = [] # x19, x20, x21, x22] +vfp_argument_regs = caller_vfp_resp = all_vfp_regs[:8] argument_regs = [x0, x1, x2, x3, x4, x5, x6, x7] caller_resp = argument_regs + [x8, x9, x10, x11, x12, x13, x14, x15] diff --git a/rpython/jit/backend/aarch64/runner.py b/rpython/jit/backend/aarch64/runner.py --- a/rpython/jit/backend/aarch64/runner.py +++ b/rpython/jit/backend/aarch64/runner.py @@ -12,6 +12,7 @@ all_reg_indexes = range(len(r.all_regs)) gen_regs = r.all_regs float_regs = VFPRegisterManager.all_regs + supports_floats = True from rpython.jit.backend.aarch64.arch import JITFRAME_FIXED_SIZE From pypy.commits at gmail.com Tue Jun 18 09:35:06 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 18 Jun 2019 06:35:06 -0700 (PDT) Subject: [pypy-commit] pypy arm64: enough for basic float operations Message-ID: <5d08e88a.1c69fb81.3e197.ff3c@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96820:254778fcde03 Date: 2019-06-18 13:34 +0000 http://bitbucket.org/pypy/pypy/changeset/254778fcde03/ Log: enough for basic float operations diff --git a/rpython/jit/backend/aarch64/arch.py b/rpython/jit/backend/aarch64/arch.py --- a/rpython/jit/backend/aarch64/arch.py +++ b/rpython/jit/backend/aarch64/arch.py @@ -8,5 +8,5 @@ # A jitframe is a jit.backend.llsupport.llmodel.jitframe.JITFRAME # Stack frame fixed area # Currently only the force_index -JITFRAME_FIXED_SIZE = 16 + 16 -# 16 GPR + 16 VFP Regs # 20 if we want to use 4 extra x19..x22 +JITFRAME_FIXED_SIZE = 16 + 8 +# 16 GPR + 8 VFP Regs # 20 if we want to use 4 extra x19..x22 diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -822,8 +822,8 @@ if value.is_imm(): self.mc.gen_load_int(loc.value, value.getint()) elif value.is_imm_float(): - self.mc.gen_load_int(r.ip.value, value.getint()) - self.mc.VLDR(loc.value, r.ip.value) + self.mc.gen_load_int(r.ip0.value, value.getint()) + self.mc.LDR_di(loc.value, r.ip0.value, 0) def _mov_stack_to_loc(self, prev_loc, loc): offset = prev_loc.value @@ -834,7 +834,13 @@ assert 0 <= offset <= (1<<15) - 1 self.mc.LDR_ri(loc.value, r.fp.value, offset) return - xxx + if loc.is_vfp_reg(): + assert prev_loc.type == FLOAT, 'trying to load from an \ + incompatible location into a float register' + assert 0 <= offset <= (1 << 15) - 1 + self.mc.LDR_di(loc.value, r.fp.value, offset) + return + assert False # elif loc.is_vfp_reg(): # assert prev_loc.type == FLOAT, 'trying to load from an \ # incompatible location into a float register' @@ -932,12 +938,8 @@ return self._store_core_reg(mc, source, base, ofs) def _store_vfp_reg(self, mc, source, base, ofs): - if check_imm_arg(ofs, VMEM_imm_size): - mc.VSTR(source.value, base.value, imm=ofs, cond=cond) - else: - mc.gen_load_int(helper.value, ofs, cond=cond) - mc.ADD_rr(helper.value, base.value, helper.value, cond=cond) - mc.VSTR(source.value, helper.value, cond=cond) + assert ofs <= (1 << 15) - 1 + mc.STR_di(source.value, base.value, ofs) def _store_core_reg(self, mc, source, base, ofs): # uses r.ip1 as a temporary diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -171,6 +171,22 @@ base = 0b10001011000 | (s << 8) self.write32((base << 21) | (rm << 16) | (rn << 5) | (rd)) + def FADD_dd(self, rd, rn, rm): + base = 0b00011110011 + self.write32((base << 21) | (rm << 16) | (0b001010 << 10) | (rn << 5) | rd) + + def FSUB_dd(self, rd, rn, rm): + base = 0b00011110011 + self.write32((base << 21) | (rm << 16) | (0b001110 << 10) | (rn << 5) | rd) + + def FMUL_dd(self, rd, rn, rm): + base = 0b00011110011 + self.write32((base << 21) | (rm << 16) | (0b000010 << 10) | (rn << 5) | rd) + + def FDIV_dd(self, rd, rn, rm): + base = 0b00011110011 + self.write32((base << 21) | (rm << 16) | (0b000110 << 10) | (rn << 5) | rd) + def SUB_rr(self, rd, rn, rm, s=0): base = 0b11001011001 | (s << 8) self.write32((base << 21) | (rm << 16) | (0b11 << 13) | (rn << 5) | (rd)) diff --git a/rpython/jit/backend/aarch64/locations.py b/rpython/jit/backend/aarch64/locations.py --- a/rpython/jit/backend/aarch64/locations.py +++ b/rpython/jit/backend/aarch64/locations.py @@ -114,6 +114,32 @@ return self.type == FLOAT +class ConstFloatLoc(AssemblerLocation): + """This class represents an imm float value which is stored in memory at + the address stored in the field value""" + _immutable_ = True + width = WORD + type = FLOAT + + def __init__(self, value): + self.value = value + + def getint(self): + return self.value + + def __repr__(self): + return "imm_float(stored at %d)" % (self.value) + + def is_imm_float(self): + return True + + def as_key(self): # a real address + 1 + return self.value | 1 + + def is_float(self): + return True + + class ZeroRegister(AssemblerLocation): _immutable_ = True diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -211,6 +211,22 @@ emit_op_cast_ptr_to_int = _genop_same_as emit_op_cast_int_to_ptr = _genop_same_as + def emit_op_float_add(self, op, arglocs): + arg1, arg2, res = arglocs + self.mc.FADD_dd(res.value, arg1.value, arg2.value) + + def emit_op_float_sub(self, op, arglocs): + arg1, arg2, res = arglocs + self.mc.FSUB_dd(res.value, arg1.value, arg2.value) + + def emit_op_float_mul(self, op, arglocs): + arg1, arg2, res = arglocs + self.mc.FMUL_dd(res.value, arg1.value, arg2.value) + + def emit_op_float_truediv(self, op, arglocs): + arg1, arg2, res = arglocs + self.mc.FDIV_dd(res.value, arg1.value, arg2.value) + def emit_op_load_from_gc_table(self, op, arglocs): res_loc, = arglocs index = op.getarg(0).getint() diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -19,11 +19,11 @@ from rpython.jit.backend.llsupport.gcmap import allocate_gcmap from rpython.jit.backend.llsupport.descr import CallDescr from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.codewriter import longlong from rpython.rlib.rarithmetic import r_uint - class TempInt(TempVar): type = INT @@ -438,7 +438,22 @@ return [reg] prepare_comp_op_int_is_true = prepare_comp_unary - prepare_comp_op_int_is_zero = prepare_comp_unary + prepare_comp_op_int_is_zero = prepare_comp_unary + + # --------------------------------- floats -------------------------- + + def prepare_two_regs_op(self, op): + loc1 = self.make_sure_var_in_reg(op.getarg(0)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) + self.possibly_free_vars_for_op(op) + self.free_temp_vars() + res = self.force_allocate_reg(op) + return [loc1, loc2, res] + + prepare_op_float_add = prepare_two_regs_op + prepare_op_float_sub = prepare_two_regs_op + prepare_op_float_mul = prepare_two_regs_op + prepare_op_float_truediv = prepare_two_regs_op # --------------------------------- fields -------------------------- From pypy.commits at gmail.com Tue Jun 18 10:03:44 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 18 Jun 2019 07:03:44 -0700 (PDT) Subject: [pypy-commit] pypy arm64: more float ops Message-ID: <5d08ef40.1c69fb81.2192e.acbc@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96821:aba1d6e3b06a Date: 2019-06-18 14:02 +0000 http://bitbucket.org/pypy/pypy/changeset/aba1d6e3b06a/ Log: more float ops diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -187,6 +187,18 @@ base = 0b00011110011 self.write32((base << 21) | (rm << 16) | (0b000110 << 10) | (rn << 5) | rd) + def FCMP_dd(self, rn, rm): + base = 0b00011110011 + self.write32((base << 21) | (rm << 16) | (0b001000 << 10) | (rn << 5)) + + def FNEG_d(self, rd, rn): + base = 0b0001111001100001010000 + self.write32((base << 10) | (rn << 5) | rd) + + def FABS_d(self, rd, rn): + base = 0b0001111001100000110000 + self.write32((base << 10) | (rn << 5) | rd) + def SUB_rr(self, rd, rn, rm, s=0): base = 0b11001011001 | (s << 8) self.write32((base << 21) | (rm << 16) | (0b11 << 13) | (rn << 5) | (rd)) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -24,6 +24,14 @@ emit_op.__name__ = name return emit_op +def gen_float_comp_op(name, flag): + def emit_op(self, op, arglocs): + l0, l1, res = arglocs + self.emit_float_comp_op(op, l0, l1) + self.mc.CSET_r_flag(res.value, c.get_opposite_of(flag)) + emit_op.__name__ = name + return emit_op + class ResOpAssembler(BaseAssembler): def int_sub_impl(self, op, arglocs, flags=0): l0, l1, res = arglocs @@ -107,6 +115,9 @@ else: self.mc.CMP_rr(l0.value, l1.value) + def emit_float_comp_op(self, op, l0, l1): + self.mc.FCMP_dd(l0.value, l1.value) + def emit_comp_op_int_lt(self, op, arglocs): self.emit_int_comp_op(op, arglocs[0], arglocs[1]) return c.LT @@ -227,6 +238,21 @@ arg1, arg2, res = arglocs self.mc.FDIV_dd(res.value, arg1.value, arg2.value) + emit_op_float_lt = gen_float_comp_op('float_lt', c.VFP_LT) + emit_op_float_le = gen_float_comp_op('float_le', c.VFP_LE) + emit_op_float_eq = gen_float_comp_op('float_eq', c.EQ) + emit_op_float_ne = gen_float_comp_op('float_ne', c.NE) + emit_op_float_gt = gen_float_comp_op('float_gt', c.GT) + emit_op_float_ge = gen_float_comp_op('float_ge', c.GE) + + def emit_op_float_neg(self, op, arglocs): + arg, res = arglocs + self.mc.FNEG_d(res.value, arg.value) + + def emit_op_float_abs(self, op, arglocs): + arg, res = arglocs + self.mc.FABS_d(res.value, arg.value) + def emit_op_load_from_gc_table(self, op, arglocs): res_loc, = arglocs index = op.getarg(0).getint() diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -455,6 +455,16 @@ prepare_op_float_mul = prepare_two_regs_op prepare_op_float_truediv = prepare_two_regs_op + prepare_op_float_lt = prepare_two_regs_op + prepare_op_float_le = prepare_two_regs_op + prepare_op_float_eq = prepare_two_regs_op + prepare_op_float_ne = prepare_two_regs_op + prepare_op_float_gt = prepare_two_regs_op + prepare_op_float_ge = prepare_two_regs_op + + prepare_op_float_neg = prepare_unary + prepare_op_float_abs = prepare_unary + # --------------------------------- fields -------------------------- def prepare_op_gc_store(self, op): From pypy.commits at gmail.com Tue Jun 18 11:14:19 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 18 Jun 2019 08:14:19 -0700 (PDT) Subject: [pypy-commit] pypy arm64: casts Message-ID: <5d08ffcb.1c69fb81.fd756.09da@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96822:9453a6ab3fe2 Date: 2019-06-18 14:57 +0000 http://bitbucket.org/pypy/pypy/changeset/9453a6ab3fe2/ Log: casts diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -199,6 +199,14 @@ base = 0b0001111001100000110000 self.write32((base << 10) | (rn << 5) | rd) + def FCVTZS_d(self, rd, rn): + base = 0b1001111001111000 + self.write32((base << 16) | (rn << 5) | rd) + + def SCVTF_r(self, rd, rn): + base = 0b1001111001100010 + self.write32((base << 16) | (rn << 5) | rd) + def SUB_rr(self, rd, rn, rm, s=0): base = 0b11001011001 | (s << 8) self.write32((base << 21) | (rm << 16) | (0b11 << 13) | (rn << 5) | (rd)) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -253,6 +253,14 @@ arg, res = arglocs self.mc.FABS_d(res.value, arg.value) + def emit_op_cast_float_to_int(self, op, arglocs): + arg, res = arglocs + self.mc.FCVTZS_d(res.value, arg.value) + + def emit_op_cast_int_to_float(self, op, arglocs): + arg, res = arglocs + self.mc.SCVTF_r(res.value, arg.value) + def emit_op_load_from_gc_table(self, op, arglocs): res_loc, = arglocs index = op.getarg(0).getint() diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -464,6 +464,9 @@ prepare_op_float_neg = prepare_unary prepare_op_float_abs = prepare_unary + prepare_op_cast_float_to_int = prepare_unary + prepare_op_cast_int_to_float = prepare_unary + # --------------------------------- fields -------------------------- From pypy.commits at gmail.com Tue Jun 18 11:14:20 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 18 Jun 2019 08:14:20 -0700 (PDT) Subject: [pypy-commit] pypy arm64: more float ops Message-ID: <5d08ffcc.1c69fb81.fcfac.1674@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96823:8e29d4e40226 Date: 2019-06-18 15:13 +0000 http://bitbucket.org/pypy/pypy/changeset/8e29d4e40226/ Log: more float ops diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -890,9 +890,15 @@ elif loc.is_stack(): self.mc.LDR_ri(r.ip0.value, r.fp.value, loc.value) self.mc.STR_ri(r.ip0.value, r.sp.value, pos) + elif loc.is_vfp_reg(): + xxx else: assert False, "wrong loc" + def _mov_imm_float_to_loc(self, prev_loc, loc): + assert loc.is_vfp_reg() + self.load(loc, prev_loc) + def regalloc_mov(self, prev_loc, loc): """Moves a value from a previous location to some other location""" if prev_loc.is_imm(): diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -343,6 +343,13 @@ assert base_loc.is_core_reg() if scale == 3: # WORD size + if value_loc.is_float(): + if ofs_loc.is_imm(): + self.mc.STR_di(value_loc.value, base_loc.value, + ofs_loc.value) + else: + xxx + return if ofs_loc.is_imm(): self.mc.STR_ri(value_loc.value, base_loc.value, ofs_loc.value) @@ -365,6 +372,12 @@ # if scale == 3: # WORD + if res_loc.is_float(): + if ofs_loc.is_imm(): + self.mc.LDR_di(res_loc.value, base_loc.value, ofs_loc.value) + else: + self.mc.LDR_dr(res_loc.value, base_loc.value, ofs_loc.value) + return if ofs_loc.is_imm(): self.mc.LDR_ri(res_loc.value, base_loc.value, ofs_loc.value) else: diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -97,6 +97,9 @@ rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = x return locations.ConstFloatLoc(adr) + def call_result_location(self, v): + return r.d0 + def __init__(self, longevity, frame_manager=None, assembler=None): RegisterManager.__init__(self, longevity, frame_manager, assembler) diff --git a/rpython/jit/backend/aarch64/registers.py b/rpython/jit/backend/aarch64/registers.py --- a/rpython/jit/backend/aarch64/registers.py +++ b/rpython/jit/backend/aarch64/registers.py @@ -23,6 +23,9 @@ callee_saved_registers = [] # x19, x20, x21, x22] vfp_argument_regs = caller_vfp_resp = all_vfp_regs[:8] +[d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12, d13, d14, + d15, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, + d28, d29, d30, d31] = vfpregisters argument_regs = [x0, x1, x2, x3, x4, x5, x6, x7] caller_resp = argument_regs + [x8, x9, x10, x11, x12, x13, x14, x15] From pypy.commits at gmail.com Tue Jun 18 11:24:33 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 18 Jun 2019 08:24:33 -0700 (PDT) Subject: [pypy-commit] pypy arm64: more floats Message-ID: <5d090231.1c69fb81.25a6a.1d1e@mx.google.com> Author: fijal Branch: arm64 Changeset: r96824:1f31b1f9873c Date: 2019-06-18 17:23 +0200 http://bitbucket.org/pypy/pypy/changeset/1f31b1f9873c/ Log: more floats diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -30,6 +30,10 @@ assert 0 <= offset < 32768 self.write32((base << 22) | ((offset >> 3) << 10) | (rn << 5) | rt) + def STR_dd(self, rt, rn, rm): + base = 0b11111100001 + self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt) + def STP_rr_preindex(self, reg1, reg2, rn, offset): base = 0b1010100110 assert -512 <= offset < 512 @@ -122,6 +126,10 @@ base = 0b1111110101 self.write32((base << 22) | (offset >> 3 << 10) | (rn << 5) | rt) + def LDR_dr(self, rt, rn, rm): + base = 0b11111100011 + self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt) + def LDRB_ri(self, rt, rn, immed): base = 0b0011100101 assert 0 <= immed <= 1<<12 diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -348,7 +348,8 @@ self.mc.STR_di(value_loc.value, base_loc.value, ofs_loc.value) else: - xxx + self.mc.STR_dd(value_loc.value, base_loc.value, + ofs_loc.value) return if ofs_loc.is_imm(): self.mc.STR_ri(value_loc.value, base_loc.value, @@ -489,17 +490,24 @@ def emit_op_guard_value(self, op, arglocs): v0 = arglocs[0] - assert v0.is_core_reg() # can be also a float reg, but later v1 = arglocs[1] - if v1.is_core_reg(): - loc = v1 - elif v1.is_imm(): - self.mc.gen_load_int(r.ip0.value, v1.value) - loc = r.ip0 + if v0.is_core_reg(): + if v1.is_core_reg(): + loc = v1 + elif v1.is_imm(): + self.mc.gen_load_int(r.ip0.value, v1.value) + loc = r.ip0 + else: + assert v1.is_stack() + yyy + self.mc.CMP_rr(v0.value, loc.value) else: - assert v1.is_stack() - yyy - self.mc.CMP_rr(v0.value, loc.value) + assert v0.is_vfp_reg() + if v1.is_vfp_reg(): + loc = v1 + else: + xxx + self.mc.FCMP_dd(v0.value, loc.value) self._emit_guard(op, c.EQ, arglocs[2:]) def emit_op_guard_class(self, op, arglocs): From pypy.commits at gmail.com Tue Jun 18 11:40:34 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 18 Jun 2019 08:40:34 -0700 (PDT) Subject: [pypy-commit] pypy arm64: minor progress Message-ID: <5d0905f2.1c69fb81.4ad2e.40d6@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96825:0abc0d2b282f Date: 2019-06-18 15:39 +0000 http://bitbucket.org/pypy/pypy/changeset/0abc0d2b282f/ Log: minor progress diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -899,6 +899,10 @@ assert loc.is_vfp_reg() self.load(loc, prev_loc) + def _mov_vfp_reg_to_loc(self, prev_loc, loc): + assert loc.is_stack() + self.mc.STR_di(prev_loc.value, r.fp.value, loc.value) + def regalloc_mov(self, prev_loc, loc): """Moves a value from a previous location to some other location""" if prev_loc.is_imm(): diff --git a/rpython/jit/backend/aarch64/locations.py b/rpython/jit/backend/aarch64/locations.py --- a/rpython/jit/backend/aarch64/locations.py +++ b/rpython/jit/backend/aarch64/locations.py @@ -118,7 +118,6 @@ """This class represents an imm float value which is stored in memory at the address stored in the field value""" _immutable_ = True - width = WORD type = FLOAT def __init__(self, value): From pypy.commits at gmail.com Tue Jun 18 15:20:17 2019 From: pypy.commits at gmail.com (rlamy) Date: Tue, 18 Jun 2019 12:20:17 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Optimise str(): don't rescan a string that is known to always be ASCII Message-ID: <5d093971.1c69fb81.11959.0048@mx.google.com> Author: Ronan Lamy Branch: py3.6 Changeset: r96827:92c90687b29c Date: 2019-06-18 19:33 +0100 http://bitbucket.org/pypy/pypy/changeset/92c90687b29c/ Log: Optimise str(): don't rescan a string that is known to always be ASCII diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -604,7 +604,7 @@ def descr_repr(self, space): res = str(self.intval) - return space.newtext(res) + return space.newutf8(res, len(res)) # res is always ASCII descr_str = func_with_new_name(descr_repr, 'descr_str') def descr_format(self, space, w_format_spec): From pypy.commits at gmail.com Wed Jun 19 06:03:22 2019 From: pypy.commits at gmail.com (arigo) Date: Wed, 19 Jun 2019 03:03:22 -0700 (PDT) Subject: [pypy-commit] cffi default: (brendan on mailing list) Ignore preprocessor stuff when looking for '"' Message-ID: <5d0a086a.1c69fb81.b1f8b.acbe@mx.google.com> Author: Armin Rigo Branch: Changeset: r3281:8097b9f9ab09 Date: 2019-06-19 11:59 +0200 http://bitbucket.org/cffi/cffi/changeset/8097b9f9ab09/ Log: (brendan on mailing list) Ignore preprocessor stuff when looking for '"' diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -145,12 +145,16 @@ return ''.join(parts) def _warn_for_string_literal(csource): - if '"' in csource: - import warnings - warnings.warn("String literal found in cdef() or type source. " - "String literals are ignored here, but you should " - "remove them anyway because some character sequences " - "confuse pre-parsing.") + if '"' not in csource: + return + for line in csource.splitlines(): + if '"' in line and not line.lstrip().startswith('#'): + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + break def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section From pypy.commits at gmail.com Wed Jun 19 13:47:10 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 19 Jun 2019 10:47:10 -0700 (PDT) Subject: [pypy-commit] pypy setitem2d: test for 2d memoryview.__setitem__, passes with -A (issue bb-3028) Message-ID: <5d0a751e.1c69fb81.6b50b.1db6@mx.google.com> Author: Matti Picus Branch: setitem2d Changeset: r96828:06767d845ae4 Date: 2019-06-19 20:46 +0300 http://bitbucket.org/pypy/pypy/changeset/06767d845ae4/ Log: test for 2d memoryview.__setitem__, passes with -A (issue bb-3028) diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -184,7 +184,7 @@ if self.view.readonly: raise oefmt(space.w_TypeError, "cannot modify read-only memory") if space.isinstance_w(w_index, space.w_tuple): - raise oefmt(space.w_NotImplementedError, "") + raise oefmt(space.w_NotImplementedError, "only 1d setitem supported") start, stop, step, size = space.decode_index4(w_index, self.getlength()) is_slice = space.isinstance_w(w_index, space.w_slice) start, stop, step, slicelength = self._decode_index(space, w_index, is_slice) diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -287,6 +287,12 @@ assert m2.itemsize == m1.itemsize assert m2.shape == m1.shape + def test_2d(self): + m = memoryview(bytearray(b'1234123412341234')).cast('b', shape=(4, 4)) + assert m[2, 3] == ord('4') + m[2, 3] = ord('z') + assert m[2, 3] == ord('z') + class AppTestCtypes(object): spaceconfig = dict(usemodules=['sys', '_rawffi']) From pypy.commits at gmail.com Wed Jun 19 16:20:10 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 19 Jun 2019 13:20:10 -0700 (PDT) Subject: [pypy-commit] pypy setitem2d: fix for for 2d memoryview.__setitem__, (issue bb-3028) Message-ID: <5d0a98fa.1c69fb81.9973b.6317@mx.google.com> Author: Matti Picus Branch: setitem2d Changeset: r96829:0f9d6d72dfe9 Date: 2019-06-19 23:19 +0300 http://bitbucket.org/pypy/pypy/changeset/0f9d6d72dfe9/ Log: fix for for 2d memoryview.__setitem__, (issue bb-3028) diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -132,6 +132,24 @@ data = view.getbytes(start, itemsize) return view.value_from_bytes(space, data) + def _setitem_tuple_indexed(self, space, w_index, w_obj): + view = self.view + length = space.len_w(w_index) + ndim = view.getndim() + if length < ndim: + raise oefmt(space.w_NotImplementedError, + "sub-views are not implemented") + + if length > ndim: + raise oefmt(space.w_TypeError, \ + "cannot index %d-dimension view with %d-element tuple", + length, ndim) + + start = self._start_from_tuple(space, w_index) + itemsize = self.getitemsize() + val = self.view.bytes_from_value(space, w_obj) + self.view.setbytes(start * itemsize, val) + def _decode_index(self, space, w_index, is_slice): shape = self.getshape() if len(shape) == 0: @@ -184,7 +202,7 @@ if self.view.readonly: raise oefmt(space.w_TypeError, "cannot modify read-only memory") if space.isinstance_w(w_index, space.w_tuple): - raise oefmt(space.w_NotImplementedError, "only 1d setitem supported") + return self._setitem_tuple_indexed(space, w_index, w_obj) start, stop, step, size = space.decode_index4(w_index, self.getlength()) is_slice = space.isinstance_w(w_index, space.w_slice) start, stop, step, slicelength = self._decode_index(space, w_index, is_slice) diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -288,10 +288,17 @@ assert m2.shape == m1.shape def test_2d(self): - m = memoryview(bytearray(b'1234123412341234')).cast('b', shape=(4, 4)) + m = memoryview(bytearray(b'1234123412341234')) + assert m[3] == ord('4') + m[3] = ord('z') + assert m[3] == ord('z') + m = m.cast('B', shape=(4, 4)) assert m[2, 3] == ord('4') - m[2, 3] = ord('z') - assert m[2, 3] == ord('z') + m[2, 3] = ord('a') + assert m[2, 3] == ord('a') + raises(TypeError, m.__setitem__, (2, 3), bytearray(b'12')) + # slices in 2d memoryviews are not supported at all + raises(TypeError, m.__getitem__, (slice(None), 3)) class AppTestCtypes(object): spaceconfig = dict(usemodules=['sys', '_rawffi']) From pypy.commits at gmail.com Wed Jun 19 18:46:40 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 19 Jun 2019 15:46:40 -0700 (PDT) Subject: [pypy-commit] pypy freelibrary: export FreeLibrary, non-closing LoadLibrary from rwin32 Message-ID: <5d0abb50.1c69fb81.8cc11.05a0@mx.google.com> Author: Matti Picus Branch: freelibrary Changeset: r96830:ec338ff9ef03 Date: 2019-06-20 01:44 +0300 http://bitbucket.org/pypy/pypy/changeset/ec338ff9ef03/ Log: export FreeLibrary, non-closing LoadLibrary from rwin32 diff --git a/lib_pypy/_ctypes/__init__.py b/lib_pypy/_ctypes/__init__.py --- a/lib_pypy/_ctypes/__init__.py +++ b/lib_pypy/_ctypes/__init__.py @@ -16,7 +16,7 @@ import os as _os if _os.name in ("nt", "ce"): - from _rawffi import FormatError + from _rawffi import FormatError, FreeLibrary, LoadLibrary from _rawffi import check_HRESULT as _check_HRESULT try: from __pypy__ import builtinify @@ -34,7 +34,6 @@ del builtinify - LoadLibrary = dlopen from _rawffi import FUNCFLAG_STDCALL, FUNCFLAG_CDECL, FUNCFLAG_PYTHONAPI from _rawffi import FUNCFLAG_USE_ERRNO, FUNCFLAG_USE_LASTERROR diff --git a/pypy/module/_rawffi/__init__.py b/pypy/module/_rawffi/__init__.py --- a/pypy/module/_rawffi/__init__.py +++ b/pypy/module/_rawffi/__init__.py @@ -31,6 +31,9 @@ 'SegfaultException' : 'space.new_exception_class("_rawffi.SegfaultException")', 'exit' : 'interp_exit.exit', } + if sys.platform == 'win32': + interpleveldefs['LoadLibrary'] = 'interp_rawffi.LoadLibrary' + interpleveldefs['FreeLibrary'] = 'interp_rawffi.FreeLibrary' appleveldefs = { } diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -658,11 +658,26 @@ if sys.platform == 'win32': # see also # https://bitbucket.org/pypy/pypy/issue/1944/ctypes-on-windows-getlasterror + def get_last_error(space): return space.newint(rwin32.GetLastError_alt_saved()) @unwrap_spec(error=int) def set_last_error(space, error): rwin32.SetLastError_alt_saved(error) + + @unwrap_spec(name='fsencode_or_none') + def LoadLibrary(space, name): + with rffi.scoped_str2charp(libname) as ll_libname: + try: + return dlopen(ll_libname) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, name or "") + except OSError as e: + raise wrap_oserror(space, e) + + def FreeLibrary(space, w_handle): + return rwin32.FreeLibrary(space.int_w(w_handle)) + else: # always have at least a dummy version of these functions # (https://bugs.pypy.org/issue1242) diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -288,11 +288,15 @@ if not self.iswin32: skip("win32 specific") import _rawffi - lib = _rawffi.CDLL(self.lib_name) - # This will call the ordinal function numbered 1 - # my compiler seems to order them alphabetically: - # AAA_first_ordinal_function - assert lib.ptr(1, [], 'i')()[0] == 42 + lib = _rawffi.LoadLibrary(self.lib_name) + try: + # This will call the ordinal function numbered 1 + # my compiler seems to order them alphabetically: + # AAA_first_ordinal_function + assert lib.ptr(1, [], 'i')()[0] == 42 + finally: + assert _rawffi.FreeLibrary(lib) == 0 + def test_getchar(self): import _rawffi From pypy.commits at gmail.com Thu Jun 20 18:42:55 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 20 Jun 2019 15:42:55 -0700 (PDT) Subject: [pypy-commit] pypy freelibrary: missing import Message-ID: <5d0c0bef.1c69fb81.d8964.3ee9@mx.google.com> Author: Matti Picus Branch: freelibrary Changeset: r96831:6ad765b702ef Date: 2019-06-20 10:44 +0300 http://bitbucket.org/pypy/pypy/changeset/6ad765b702ef/ Log: missing import diff --git a/pypy/module/_rawffi/__init__.py b/pypy/module/_rawffi/__init__.py --- a/pypy/module/_rawffi/__init__.py +++ b/pypy/module/_rawffi/__init__.py @@ -1,6 +1,7 @@ """ Low-level interface to clibffi """ +import sys from pypy.interpreter.mixedmodule import MixedModule from pypy.module._rawffi import alt From pypy.commits at gmail.com Thu Jun 20 18:42:57 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 20 Jun 2019 15:42:57 -0700 (PDT) Subject: [pypy-commit] pypy setitem2d: document and close branch Message-ID: <5d0c0bf1.1c69fb81.b3984.4700@mx.google.com> Author: Matti Picus Branch: setitem2d Changeset: r96832:50bd9ba82535 Date: 2019-06-21 01:40 +0300 http://bitbucket.org/pypy/pypy/changeset/50bd9ba82535/ Log: document and close branch diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -12,3 +12,7 @@ .. branch: winmultiprocessing Improve multiprocessing support on win32 + +.. branch: setitem2d + +Allow 2d indexing in ``memoryview.__setitem__`` (issue bb-3028) From pypy.commits at gmail.com Thu Jun 20 18:42:58 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 20 Jun 2019 15:42:58 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: merge setitem2d which allows more than 1d indexing of memoryview.__setitem__ Message-ID: <5d0c0bf2.1c69fb81.9f344.9ee9@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96833:da82c5ed6895 Date: 2019-06-21 01:42 +0300 http://bitbucket.org/pypy/pypy/changeset/da82c5ed6895/ Log: merge setitem2d which allows more than 1d indexing of memoryview.__setitem__ diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -12,3 +12,7 @@ .. branch: winmultiprocessing Improve multiprocessing support on win32 + +.. branch: setitem2d + +Allow 2d indexing in ``memoryview.__setitem__`` (issue bb-3028) diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -132,6 +132,24 @@ data = view.getbytes(start, itemsize) return view.value_from_bytes(space, data) + def _setitem_tuple_indexed(self, space, w_index, w_obj): + view = self.view + length = space.len_w(w_index) + ndim = view.getndim() + if length < ndim: + raise oefmt(space.w_NotImplementedError, + "sub-views are not implemented") + + if length > ndim: + raise oefmt(space.w_TypeError, \ + "cannot index %d-dimension view with %d-element tuple", + length, ndim) + + start = self._start_from_tuple(space, w_index) + itemsize = self.getitemsize() + val = self.view.bytes_from_value(space, w_obj) + self.view.setbytes(start * itemsize, val) + def _decode_index(self, space, w_index, is_slice): shape = self.getshape() if len(shape) == 0: @@ -184,7 +202,7 @@ if self.view.readonly: raise oefmt(space.w_TypeError, "cannot modify read-only memory") if space.isinstance_w(w_index, space.w_tuple): - raise oefmt(space.w_NotImplementedError, "") + return self._setitem_tuple_indexed(space, w_index, w_obj) start, stop, step, size = space.decode_index4(w_index, self.getlength()) is_slice = space.isinstance_w(w_index, space.w_slice) start, stop, step, slicelength = self._decode_index(space, w_index, is_slice) diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -287,6 +287,19 @@ assert m2.itemsize == m1.itemsize assert m2.shape == m1.shape + def test_2d(self): + m = memoryview(bytearray(b'1234123412341234')) + assert m[3] == ord('4') + m[3] = ord('z') + assert m[3] == ord('z') + m = m.cast('B', shape=(4, 4)) + assert m[2, 3] == ord('4') + m[2, 3] = ord('a') + assert m[2, 3] == ord('a') + raises(TypeError, m.__setitem__, (2, 3), bytearray(b'12')) + # slices in 2d memoryviews are not supported at all + raises(TypeError, m.__getitem__, (slice(None), 3)) + class AppTestCtypes(object): spaceconfig = dict(usemodules=['sys', '_rawffi']) From pypy.commits at gmail.com Fri Jun 21 02:46:16 2019 From: pypy.commits at gmail.com (mattip) Date: Thu, 20 Jun 2019 23:46:16 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: more closely follow cpython implementation for error modes Message-ID: <5d0c7d38.1c69fb81.6ef55.9e56@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96834:bfa7f1269de9 Date: 2019-06-21 09:45 +0300 http://bitbucket.org/pypy/pypy/changeset/bfa7f1269de9/ Log: more closely follow cpython implementation for error modes diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -18,6 +18,31 @@ MEMORYVIEW_SCALAR = 0x0008 MEMORYVIEW_PIL = 0x0010 +def is_multiindex(space, w_key): + if not space.isinstance_w(w_key, space.w_tuple): + return 0 + length = space.len_w(w_key) + i = 0 + while i < length: + w_obj = w_key.getitem(space, i) + if not space.lookup(w_obj, '__index__'): + return 0 + i += 1 + return 1 + +def is_multislice(space, w_key): + if not space.isinstance_w(w_key, space.w_tuple): + return 0 + length = space.len_w(w_key) + if length == 0: + return 0 + i = 0 + while i < length: + w_obj = w_key.getitem(space, i) + if not space.isinstance_w(w_obj, space.w_slice): + return 0 + i += 1 + return 1 class W_MemoryView(W_Root): """Implement the built-in 'memoryview' type as a wrapper around @@ -161,24 +186,28 @@ def descr_getitem(self, space, w_index): self._check_released(space) - if space.isinstance_w(w_index, space.w_tuple): + is_slice = space.isinstance_w(w_index, space.w_slice) + if is_slice or space.lookup(w_index, '__index__'): + start, stop, step, slicelength = self._decode_index(space, w_index, is_slice) + # ^^^ for a non-slice index, this returns (index, 0, 0, 1) + if step == 0: # index only + dim = self.getndim() + if dim == 0: + raise oefmt(space.w_TypeError, "invalid indexing of 0-dim memory") + elif dim == 1: + return self.view.w_getitem(space, start) + else: + raise oefmt(space.w_NotImplementedError, + "multi-dimensional sub-views are not implemented") + elif is_slice: + return self.view.new_slice(start, step, slicelength).wrap(space) + elif is_multiindex(space, w_index): return self._getitem_tuple_indexed(space, w_index) - is_slice = space.isinstance_w(w_index, space.w_slice) - start, stop, step, slicelength = self._decode_index(space, w_index, is_slice) - # ^^^ for a non-slice index, this returns (index, 0, 0, 1) - if step == 0: # index only - dim = self.getndim() - if dim == 0: - raise oefmt(space.w_TypeError, "invalid indexing of 0-dim memory") - elif dim == 1: - return self.view.w_getitem(space, start) - else: - raise oefmt(space.w_NotImplementedError, "multi-dimensional sub-views are not implemented") - elif is_slice: - return self.view.new_slice(start, step, slicelength).wrap(space) - # multi index is handled at the top of this function + elif is_multislice(space, w_index): + raise oefmt(space.w_NotImplementedError, + "multi-dimensional slicing is not implemented") else: - raise TypeError("memoryview: invalid slice key") + raise oefmt(space.w_TypeError, "memoryview: invalid slice key") def init_len(self): self.length = self.bytecount_from_shape() diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -299,6 +299,7 @@ raises(TypeError, m.__setitem__, (2, 3), bytearray(b'12')) # slices in 2d memoryviews are not supported at all raises(TypeError, m.__getitem__, (slice(None), 3)) + raises(NotImplementedError, m.__getitem__, (slice(None),)) class AppTestCtypes(object): spaceconfig = dict(usemodules=['sys', '_rawffi']) From pypy.commits at gmail.com Fri Jun 21 16:06:20 2019 From: pypy.commits at gmail.com (stevie_92) Date: Fri, 21 Jun 2019 13:06:20 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-gc-cycle: Implemented tuple untracking for rrc objects Message-ID: <5d0d38bc.1c69fb81.9a7bf.284d@mx.google.com> Author: Stefan Beyer Branch: cpyext-gc-cycle Changeset: r96835:a1dc8b9e7d98 Date: 2019-06-21 22:05 +0200 http://bitbucket.org/pypy/pypy/changeset/a1dc8b9e7d98/ Log: Implemented tuple untracking for rrc objects Improved stability list traversal and removed unneccesary debug output Fixed inheritance issues with some cypthon slots diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1216,6 +1216,15 @@ state.C._PyPy_init_pyobj_list = rffi.llexternal( '_PyPy_init_pyobj_list', [], PyGC_HeadPtr, compilation_info=eci, _nowrapper=True) + state.C._PyPy_init_tuple_list = rffi.llexternal( + '_PyPy_init_tuple_list', [], PyGC_HeadPtr, + compilation_info=eci, _nowrapper=True) + state.C._PyTuple_MaybeUntrack = rffi.llexternal( + '_PyTuple_MaybeUntrack', [PyObject], lltype.Signed, + compilation_info=eci, _nowrapper=True) + state.C._PyList_CheckExact = rffi.llexternal( + '_PyList_CheckExact', [PyObject], lltype.Signed, + compilation_info=eci, _nowrapper=True) state.C._PyPy_gc_as_pyobj = rffi.llexternal( '_PyPy_gc_as_pyobj', [PyGC_HeadPtr], GCHdr_PyObject, compilation_info=eci, _nowrapper=True) @@ -1348,7 +1357,9 @@ # initialize the pyobj_list for the gc pyobj_list = space.fromcache(State).C._PyPy_init_pyobj_list() + pyobj_tuple_list = space.fromcache(State).C._PyPy_init_tuple_list() rawrefcount._init_pyobj_list(pyobj_list) + rawrefcount._init_pyobj_list(pyobj_tuple_list) # we need to call this *after* the init code above, because it might # indirectly call some functions which are attached to pypyAPI (e.g., we @@ -1552,6 +1563,7 @@ source_dir / "object.c", source_dir / "typeobject.c", source_dir / "tupleobject.c", + source_dir / "listobject.c", ] def build_eci(code, use_micronumpy=False, translating=False): diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h --- a/pypy/module/cpyext/include/listobject.h +++ b/pypy/module/cpyext/include/listobject.h @@ -1,4 +1,16 @@ -/* empty */ +#ifndef Py_LISTOBJECT_H +#define Py_LISTOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + #define PyList_Check(op) \ PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LIST_SUBCLASS) #define PyList_CheckExact(op) ((op)->ob_type == &PyList_Type) + +PyAPI_FUNC(Py_ssize_t) _PyList_CheckExact(PyObject *); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_LISTOBJECT_H */ \ No newline at end of file diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -295,6 +295,7 @@ ( (type *) _PyObject_GC_NewVar(typeobj, size) ) extern PyGC_Head *_pypy_rawrefcount_pyobj_list; +extern PyGC_Head *_pypy_rawrefcount_tuple_list; #define _Py_AS_GC(o) ((PyGC_Head *)(o)-1) #define _Py_FROM_GC(g) ((PyObject *)(((PyGC_Head *)g)+1)) @@ -327,9 +328,15 @@ #define _PyGC_REFS_REACHABLE (-3) #define _PyGC_REFS_TENTATIVELY_UNREACHABLE (-4) +#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +#define PyObject_IS_GC(o) \ + (PyType_IS_GC(Py_TYPE(o)) \ + && (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o))) + #define _PyGC_IS_TRACKED(o) (_PyGC_REFS(o) != _PyGC_REFS_UNTRACKED) - -#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +#define _PyObject_GC_MAY_BE_TRACKED(obj) \ + (PyObject_IS_GC(obj) && \ + (!PyTuple_CheckExact(obj) || _PyGC_IS_TRACKED(obj))) PyAPI_FUNC(void) PyObject_GC_Track(void *); PyAPI_FUNC(void) PyObject_GC_UnTrack(void *); @@ -344,6 +351,16 @@ ((PyGC_Head *)g->gc_prev)->gc_next = g; \ _pypy_rawrefcount_pyobj_list->gc_prev = g; \ } while(0) + #define _PyObject_GC_TRACK_Tuple(o) do { \ + PyGC_Head *g = _Py_AS_GC(o); \ + if (_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED) \ + Py_FatalError("GC object already tracked"); \ + _PyGCHead_SET_REFS(g, _PyGC_REFS_REACHABLE); \ + g->gc_next = _pypy_rawrefcount_tuple_list; \ + g->gc_prev = _pypy_rawrefcount_tuple_list->gc_prev; \ + ((PyGC_Head *)g->gc_prev)->gc_next = g; \ + _pypy_rawrefcount_tuple_list->gc_prev = g; \ + } while(0) #define _PyObject_GC_UNTRACK(o) do { \ PyGC_Head *g = _Py_AS_GC(o); \ assert(_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED); \ @@ -438,6 +455,7 @@ PyAPI_FUNC(void) _PyPy_subtype_dealloc(PyObject *); PyAPI_FUNC(void) _PyPy_object_dealloc(PyObject *); PyAPI_FUNC(PyGC_Head *) _PyPy_init_pyobj_list(); +PyAPI_FUNC(PyGC_Head *) _PyPy_init_tuple_list(); PyAPI_FUNC(GCHdr_PyObject *) _PyPy_gc_as_pyobj(PyGC_Head *); PyAPI_FUNC(PyGC_Head *) _PyPy_pyobj_as_gc(GCHdr_PyObject *); PyAPI_FUNC(Py_ssize_t) _PyPy_finalizer_type(PyGC_Head *); diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -20,6 +20,7 @@ PyAPI_FUNC(void) _PyPy_tuple_dealloc(PyObject *); PyAPI_FUNC(void) _PyPy_tuple_free(void *); PyAPI_FUNC(int) _PyPy_tuple_traverse(PyObject *ob, visitproc visit, void *arg); +PyAPI_FUNC(Py_ssize_t) _PyTuple_MaybeUntrack(PyObject *); /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); diff --git a/pypy/module/cpyext/src/listobject.c b/pypy/module/cpyext/src/listobject.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/src/listobject.c @@ -0,0 +1,7 @@ +#include "Python.h" + +Py_ssize_t +_PyList_CheckExact(PyObject *op) +{ + return op->ob_type == &PyList_Type; +} \ No newline at end of file diff --git a/pypy/module/cpyext/src/object.c b/pypy/module/cpyext/src/object.c --- a/pypy/module/cpyext/src/object.c +++ b/pypy/module/cpyext/src/object.c @@ -23,7 +23,10 @@ void* _pypy_rawrefcount_w_marker_deallocating = (void*) 0xDEADFFF; static PyGC_Head _internal_pyobj_list; +static PyGC_Head _internal_tuple_list; + PyGC_Head *_pypy_rawrefcount_pyobj_list = &_internal_pyobj_list; +PyGC_Head *_pypy_rawrefcount_tuple_list = &_internal_tuple_list; PyGC_Head * _PyPy_init_pyobj_list() @@ -33,6 +36,14 @@ return _pypy_rawrefcount_pyobj_list; } +PyGC_Head * +_PyPy_init_tuple_list() +{ + _pypy_rawrefcount_tuple_list->gc_next = _pypy_rawrefcount_tuple_list; + _pypy_rawrefcount_tuple_list->gc_prev = _pypy_rawrefcount_tuple_list; + return _pypy_rawrefcount_tuple_list; +} + GCHdr_PyObject * _PyPy_gc_as_pyobj(PyGC_Head *g) { diff --git a/pypy/module/cpyext/src/tupleobject.c b/pypy/module/cpyext/src/tupleobject.c --- a/pypy/module/cpyext/src/tupleobject.c +++ b/pypy/module/cpyext/src/tupleobject.c @@ -56,7 +56,7 @@ } for (i=0; i < size; i++) op->ob_item[i] = NULL; - _PyObject_GC_TRACK(op); + _PyObject_GC_TRACK_Tuple(op); return (PyObject *) op; } @@ -105,4 +105,33 @@ for (i = Py_SIZE(o); --i >= 0; ) Py_VISIT(o->ob_item[i]); return 0; +} + +/* Return 0 if the tuple is untracked afterwards, return 1 if the tuple + should always be kept tracked and return 2 if the tuple was not fully + intialized yet. */ +Py_ssize_t +_PyTuple_MaybeUntrack(PyObject *op) +{ + PyTupleObject *t; + Py_ssize_t i, n; + + if (!PyTuple_CheckExact(op)) + return 1; + if (!_PyGC_IS_TRACKED(op)) + return 0; + t = (PyTupleObject *) op; + n = Py_SIZE(t); + for (i = 0; i < n; i++) { + PyObject *elt = PyTuple_GET_ITEM(t, i); + /* Tuple with NULL elements aren't + fully constructed, don't untrack + them yet. */ + if (!elt) + return 2; + if (_PyObject_GC_MAY_BE_TRACKED(elt)) + return 1; + } + _PyObject_GC_UNTRACK(op); + return 0; } \ No newline at end of file diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -184,47 +184,28 @@ from pypy.module.cpyext.listobject import list_traverse # convert to pointers with correct types (PyObject) - callback_addr = llmemory.cast_ptr_to_adr(callback) - callback_ptr = llmemory.cast_adr_to_ptr(callback_addr, - visitproc) pyobj_addr = llmemory.cast_ptr_to_adr(pyobj_ptr) pyobj = llmemory.cast_adr_to_ptr(pyobj_addr, PyObject) + # now call tp_traverse (if possible) debug_print("rrc check traverse", pyobj) - pto = pyobj.c_ob_type - if pto and pto.c_tp_name: - tp_name = pto.c_tp_name - name = rffi.charp2str(cts.cast('char*', tp_name)) - debug_print("rrc try traverse", pyobj, ": type", pto, - ": name", name) - pto2 = pto - i = 1 - while pto2.c_tp_base: - base = pto2.c_tp_base - if base.c_tp_name: - tp_name = base.c_tp_name - name = rffi.charp2str(cts.cast('char*', tp_name)) - debug_print(" " * i * 3, "basetype", - base, ": name", - name, "traverse", - base.c_tp_traverse) - else: - debug_print(" " * i * 3, "unknown base") - pto2 = base - i += 1 - if pyobj.c_ob_pypy_link != 0: # special traverse - w_obj = from_ref(space, pyobj) - w_obj_type = space.type(w_obj) - if space.is_w(w_obj_type, space.w_list): # list - debug_print('rrc list traverse ', pyobj) - list_traverse(space, w_obj, callback, args) - return - - if pto and pto.c_tp_traverse: - debug_print("rrc do traverse", pyobj) - generic_cpy_call(space, pto.c_tp_traverse, pyobj, - callback_ptr, args) + # special traverse for list + if self.C._PyList_CheckExact(pyobj) != 0: + if pyobj.c_ob_pypy_link != 0: + w_obj = from_ref(space, pyobj) + if w_obj: + debug_print('rrc list traverse ', pyobj) + list_traverse(space, w_obj, callback, args) + else: + pto = pyobj.c_ob_type + if pto and pto.c_tp_traverse: + callback_addr = llmemory.cast_ptr_to_adr(callback) + callback_ptr = llmemory.cast_adr_to_ptr( + callback_addr, visitproc) + debug_print("rrc do traverse", pyobj) + generic_cpy_call(space, pto.c_tp_traverse, pyobj, + callback_ptr, args) self.tp_traverse = (lambda o, v, a:_tp_traverse(o, v, a)) @@ -258,16 +239,18 @@ # This must be called in RPython, the untranslated version # does something different. Sigh. pypyobj_list = self.C._PyPy_init_pyobj_list() + pypyobj_tuple_list = self.C._PyPy_init_tuple_list() rawrefcount.init( llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER, self.dealloc_trigger), llhelper(rawrefcount.RAWREFCOUNT_TRAVERSE, self.tp_traverse), - pypyobj_list, + pypyobj_list, pypyobj_tuple_list, self.C._PyPy_gc_as_pyobj, self.C._PyPy_pyobj_as_gc, self.C._PyPy_finalizer_type, llhelper(rawrefcount.RAWREFCOUNT_CLEAR_WR_TYPE, - self.clear_weakref_callbacks)) + self.clear_weakref_callbacks), + self.C._PyTuple_MaybeUntrack) self.builder.attach_all(space) setup_new_method_def(space) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -740,6 +740,10 @@ pto.c_tp_clear = base.c_tp_clear if not pto.c_tp_traverse: pto.c_tp_traverse = base.c_tp_traverse + if not pto.c_tp_is_gc: + pto.c_tp_is_gc = base.c_tp_is_gc + if not pto.c_tp_finalize: + pto.c_tp_finalize = base.c_tp_finalize # XXX check for correct GC flags! if not pto.c_tp_free: pto.c_tp_free = base.c_tp_free diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -3114,6 +3114,8 @@ lltype.Signed)) RAWREFCOUNT_CLEAR_WR_TYPE = lltype.Ptr(lltype.FuncType([llmemory.GCREF], lltype.Void)) + RAWREFCOUNT_MAYBE_UNTRACK_TUPLE = \ + lltype.Ptr(lltype.FuncType([PYOBJ_HDR_PTR], lltype.Signed)) RAWREFCOUNT_FINALIZER_NONE = 0 RAWREFCOUNT_FINALIZER_MODERN = 1 RAWREFCOUNT_FINALIZER_LEGACY = 2 @@ -3127,8 +3129,9 @@ return llmemory.cast_adr_to_ptr(pygchdraddr, self.PYOBJ_GC_HDR_PTR) def rawrefcount_init(self, dealloc_trigger_callback, tp_traverse, - pyobj_list, gc_as_pyobj, pyobj_as_gc, finalizer_type, - clear_weakref_callback): + pyobj_list, tuple_list, gc_as_pyobj, pyobj_as_gc, + finalizer_type, clear_weakref_callback, + tuple_maybe_untrack): # see pypy/doc/discussion/rawrefcount.rst if not self.rrc_enabled: self.rrc_p_list_young = self.AddressStack() @@ -3141,6 +3144,7 @@ self.rrc_dealloc_pending = self.AddressStack() self.rrc_tp_traverse = tp_traverse self.rrc_pyobj_list = self._pygchdr(pyobj_list) + self.rrc_tuple_list = self._pygchdr(tuple_list) self.rrc_pyobj_old_list = self._rrc_gc_list_new() self.rrc_pyobj_isolate_list = self._rrc_gc_list_new() self.rrc_pyobj_dead_list = self._rrc_gc_list_new() @@ -3150,6 +3154,7 @@ self.rrc_pyobj_as_gc = pyobj_as_gc self.rrc_finalizer_type = finalizer_type self.rrc_clear_weakref_callback = clear_weakref_callback + self.rrc_tuple_maybe_untrack = tuple_maybe_untrack self.rrc_enabled = True self.rrc_cycle_enabled = True self.rrc_state = self.RAWREFCOUNT_STATE_DEFAULT @@ -3226,6 +3231,10 @@ gchdr = self._rrc_gc_list_pop(self.rrc_pyobj_isolate_list) self._rrc_gc_list_add(self.rrc_pyobj_old_list, gchdr) return llmemory.cast_ptr_to_adr(self.rrc_gc_as_pyobj(gchdr)) +# if not self._rrc_gc_list_is_empty(self.rrc_tuple_isolate_list): +# gchdr = self._rrc_gc_list_pop(self.rrc_tuple_isolate_list) +# self._rrc_gc_list_add(self.rrc_tuple_old_list, gchdr) +# return llmemory.cast_ptr_to_adr(self.rrc_gc_as_pyobj(gchdr)) return llmemory.NULL def rawrefcount_cyclic_garbage_head(self): @@ -3288,6 +3297,8 @@ if self.rrc_enabled and (self.rrc_dealloc_pending.non_empty() or not self._rrc_gc_list_is_empty( self.rrc_pyobj_isolate_list) or +# not self._rrc_gc_list_is_empty( +# self.rrc_tuple_isolate_list) or not self._rrc_gc_list_is_empty( self.rrc_pyobj_dead_list) or not self._rrc_gc_list_is_empty( @@ -3415,6 +3426,10 @@ if not self.rrc_cycle_enabled: self._rrc_debug_check_consistency(print_label="begin-mark") + # First, untrack all tuples with only non-gc rrc objects and promote + # all other tuples to the pyobj_list + self._rrc_untrack_tuples() + # Only trace and mark rawrefcounted object if we are not doing # something special, like building gc.garbage. if (self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT and @@ -3424,7 +3439,7 @@ if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): merged_old_list = self._rrc_check_finalizer() # collect all rawrefcounted roots - self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_list) + self._rrc_collect_roots(self.rrc_pyobj_list) if merged_old_list: # set all refcounts to zero for objects in dead list # (might have been incremented) by fix_refcnt @@ -3449,13 +3464,7 @@ # now mark all pypy objects at the border, depending on the results debug_print("use_cylicrc", use_cylicrc) self.rrc_p_list_old.foreach(self._rrc_major_trace, use_cylicrc) - - # TODO: check again, if some new border objects have been marked and - # continue marking recursively... why needed? -> wrapper for - # pypy-obj is no pygc-obj??? ...KI - self._rrc_debug_check_consistency(print_label="end-mark") - #self.rrc_o_list_old.foreach(self._rrc_major_trace, use_cylicrc) # TODO: remove? def _rrc_major_trace(self, pyobject, use_cylicrefcnt): from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY @@ -3513,12 +3522,12 @@ def rrc_major_collection_free(self): if self.rrc_state == self.RAWREFCOUNT_STATE_DEFAULT: + self._rrc_debug_check_consistency() if not self._rrc_gc_list_is_empty(self.rrc_pyobj_old_list): - self._rrc_debug_check_consistency() self._rrc_clear_weakref_callbacks() self._rrc_gc_list_merge(self.rrc_pyobj_old_list, self.rrc_pyobj_dead_list) - self._rrc_debug_check_consistency(print_label="before-sweep") + self._rrc_debug_check_consistency(print_label="before-sweep") ll_assert(self.rrc_p_dict_nurs.length() == 0, "p_dict_nurs not empty 2") length_estimate = self.rrc_p_dict.length() @@ -3543,12 +3552,26 @@ # Look for any weakrefs within the trash cycle and remove the callback. # This is only needed for weakrefs created from rawrefcounted objects # because weakrefs from gc-managed objects are going away anyway. - gchdr = self.rrc_pyobj_old_list.c_gc_next - while gchdr <> self.rrc_pyobj_old_list: + list = self.rrc_pyobj_old_list + gchdr = list.c_gc_next + while gchdr <> list: pyobj = self.rrc_gc_as_pyobj(gchdr) self._rrc_traverse_weakref(pyobj) gchdr = gchdr.c_gc_next + def _rrc_untrack_tuples(self): + gchdr = self.rrc_tuple_list.c_gc_next + while gchdr <> self.rrc_tuple_list: + gchdr_next = gchdr.c_gc_next + pyobj = self.rrc_gc_as_pyobj(gchdr) + result = self.rrc_tuple_maybe_untrack(pyobj) + if result == 1: # contains gc objects -> promote to pyobj list + next = gchdr.c_gc_next + next.c_gc_prev = gchdr.c_gc_prev + gchdr.c_gc_prev.c_gc_next = next + self._rrc_gc_list_add(self.rrc_pyobj_list, gchdr) + gchdr = gchdr_next + def _rrc_visit_weakref(pyobj, self_ptr): from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance # @@ -3591,11 +3614,31 @@ else: self._rrc_free(pyobject, True) - def _rrc_collect_rawrefcount_roots(self, pygclist): + def _rrc_collect_roots(self, pygclist): + # Initialize the cyclic refcount with the real refcount. + self._rrc_collect_roots_init_list(pygclist) + + # For all non-gc pyobjects which have a refcount > 0, + # mark all reachable objects on the pypy side + self.rrc_p_list_old.foreach(self._rrc_major_trace_nongc, None) + + # For every object in this set, if it is marked, add 1 as a real + # refcount (p_list => pyobj stays alive if obj stays alive). + self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None) + self.rrc_o_list_old.foreach(self._rrc_obj_fix_refcnt, None) + + # Subtract all internal refcounts from the cyclic refcount + # of rawrefcounted objects + self._rrc_collect_roots_subtract_internal(pygclist) + + # now all rawrefcounted roots or live border objects have a + # refcount > 0 + self._rrc_debug_check_consistency(print_label="rc-initialized") + + + def _rrc_collect_roots_init_list(self, pygclist): from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT - # - # Initialize the cyclic refcount with the real refcount. pygchdr = pygclist.c_gc_next while pygchdr <> pygclist: refcnt = self.rrc_gc_as_pyobj(pygchdr).c_ob_refcnt @@ -3606,28 +3649,13 @@ self._rrc_pyobj_gc_refcnt_set(pygchdr, refcnt) pygchdr = pygchdr.c_gc_next - # For all non-gc pyobjects which have a refcount > 0, - # mark all reachable objects on the pypy side - self.rrc_p_list_old.foreach(self._rrc_major_trace_nongc, None) - - # For every object in this set, if it is marked, add 1 as a real - # refcount (p_list => pyobj stays alive if obj stays alive). - self.rrc_p_list_old.foreach(self._rrc_obj_fix_refcnt, None) - self.rrc_o_list_old.foreach(self._rrc_obj_fix_refcnt, None) - - self._rrc_debug_check_consistency(print_label="rc-initialized") - - # Subtract all internal refcounts from the cyclic refcount - # of rawrefcounted objects + def _rrc_collect_roots_subtract_internal(self, pygclist): pygchdr = pygclist.c_gc_next while pygchdr <> pygclist: pyobj = self.rrc_gc_as_pyobj(pygchdr) self._rrc_traverse(pyobj, -1) pygchdr = pygchdr.c_gc_next - # now all rawrefcounted roots or live border objects have a - # refcount > 0 - def _rrc_pyobj_gc_refcnt_set(self, pygchdr, refcnt): pygchdr.c_gc_refs &= self.RAWREFCOUNT_REFS_MASK_FINALIZED pygchdr.c_gc_refs |= refcnt << self.RAWREFCOUNT_REFS_SHIFT @@ -3662,49 +3690,54 @@ def _rrc_mark_rawrefcount(self): if self._rrc_gc_list_is_empty(self.rrc_pyobj_list): self._rrc_gc_list_init(self.rrc_pyobj_old_list) - return + else: + self._rrc_gc_list_move(self.rrc_pyobj_list, + self.rrc_pyobj_old_list) # as long as new objects with cyclic a refcount > 0 or alive border # objects are found, increment the refcount of all referenced objects # of those newly found objects - self._rrc_gc_list_move(self.rrc_pyobj_list, self.rrc_pyobj_old_list) found_alive = True + pyobj_old = self.rrc_pyobj_list # while found_alive: found_alive = False gchdr = self.rrc_pyobj_old_list.c_gc_next while gchdr <> self.rrc_pyobj_old_list: next_old = gchdr.c_gc_next - alive = (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0 - pyobj = self.rrc_gc_as_pyobj(gchdr) - if pyobj.c_ob_pypy_link <> 0: - intobj = pyobj.c_ob_pypy_link - obj = llmemory.cast_int_to_adr(intobj) - if not alive and self.header(obj).tid & ( - GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS): - # add fake refcount, to mark it as live - gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT - alive = True - if alive: - # remove from old list - next = gchdr.c_gc_next - next.c_gc_prev = gchdr.c_gc_prev - gchdr.c_gc_prev.c_gc_next = next - # add to new list - self._rrc_gc_list_add(self.rrc_pyobj_list, gchdr) - # increment refcounts - self._rrc_traverse(pyobj, 1) - # mark recursively, if it is a pypyobj - if pyobj.c_ob_pypy_link <> 0: - intobj = pyobj.c_ob_pypy_link - obj = llmemory.cast_int_to_adr(intobj) - self.objects_to_trace.append(obj) - self.visit_all_objects() - found_alive = True + found_alive |= self._rrc_mark_rawrefcount_obj(gchdr, pyobj_old) gchdr = next_old # # now all rawrefcounted objects, which are alive, have a cyclic # refcount > 0 or are marked + def _rrc_mark_rawrefcount_obj(self, gchdr, gchdr_move): + alive = (gchdr.c_gc_refs >> self.RAWREFCOUNT_REFS_SHIFT) > 0 + pyobj = self.rrc_gc_as_pyobj(gchdr) + if pyobj.c_ob_pypy_link <> 0: + intobj = pyobj.c_ob_pypy_link + obj = llmemory.cast_int_to_adr(intobj) + if not alive and self.header(obj).tid & ( + GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS): + # add fake refcount, to mark it as live + gchdr.c_gc_refs += 1 << self.RAWREFCOUNT_REFS_SHIFT + alive = True + if alive: + # remove from old list + next = gchdr.c_gc_next + next.c_gc_prev = gchdr.c_gc_prev + gchdr.c_gc_prev.c_gc_next = next + # add to new list (or not, if it is a tuple) + self._rrc_gc_list_add(gchdr_move, gchdr) + # increment refcounts + self._rrc_traverse(pyobj, 1) + # mark recursively, if it is a pypyobj + if pyobj.c_ob_pypy_link <> 0: + intobj = pyobj.c_ob_pypy_link + obj = llmemory.cast_int_to_adr(intobj) + self.objects_to_trace.append(obj) + self.visit_all_objects() + return alive + def _rrc_find_garbage(self): found_garbage = False gchdr = self.rrc_pyobj_old_list.c_gc_next @@ -3788,7 +3821,7 @@ # Check, if the cyclic isolate from the last collection cycle # is reachable from outside, after the finalizers have been # executed. - self._rrc_collect_rawrefcount_roots(self.rrc_pyobj_old_list) + self._rrc_collect_roots(self.rrc_pyobj_old_list) found_alive = False gchdr = self.rrc_pyobj_old_list.c_gc_next while gchdr <> self.rrc_pyobj_old_list: @@ -3899,6 +3932,8 @@ debug_start("rrc-lists " + print_label) self._rrc_debug_check_list(self.rrc_pyobj_list, should_print, "rrc_pyobj_list") + self._rrc_debug_check_list(self.rrc_tuple_list, + should_print, "rrc_tuple_list") self._rrc_debug_check_list(self.rrc_pyobj_old_list, should_print, "rrc_pyobj_old_list") self._rrc_debug_check_list(self.rrc_pyobj_dead_list, diff --git a/rpython/memory/gc/test/dot/free_cpython_simple.dot b/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot copy from rpython/memory/gc/test/dot/free_cpython_simple.dot copy to rpython/memory/gc/test/dot/free_cpython_tuple_1.dot --- a/rpython/memory/gc/test/dot/free_cpython_simple.dot +++ b/rpython/memory/gc/test/dot/free_cpython_tuple_1.dot @@ -1,5 +1,5 @@ digraph G { - "a" [type=C, alive=n]; + "a" [type=C, alive=n, tuple=y]; "b" [type=C, alive=n]; "a" -> "b"; "b" -> "a"; diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py --- a/rpython/memory/gc/test/test_rawrefcount.py +++ b/rpython/memory/gc/test/test_rawrefcount.py @@ -78,17 +78,32 @@ weakref.callback_cleared = True cleared = True + def rawrefcount_tuple_maybe_untrack(obj): + #if foo: + # gchdr = rawrefcount_pyobj_as_gc(obj) + # next = gchdr.c_gc_next + # next.c_gc_prev = gchdr.c_gc_prev + # gchdr.c_gc_prev.c_gc_next = next + # return 0 + return 1 # TODO: add tests for 0 ("plain" tuple) and 2 (uninitialized) + self.pyobj_list = lltype.malloc(PYOBJ_GC_HDR_PTR.TO, flavor='raw', immortal=True) self.pyobj_list.c_gc_next = self.pyobj_list self.pyobj_list.c_gc_prev = self.pyobj_list + self.tuple_list = lltype.malloc(PYOBJ_GC_HDR_PTR.TO, flavor='raw', + immortal=True) + self.tuple_list.c_gc_next = self.tuple_list + self.tuple_list.c_gc_prev = self.tuple_list self.gc.rawrefcount_init(lambda: self.trigger.append(1), rawrefcount_tp_traverse, llmemory.cast_ptr_to_adr(self.pyobj_list), + llmemory.cast_ptr_to_adr(self.tuple_list), rawrefcount_gc_as_pyobj, rawrefcount_pyobj_as_gc, rawrefcount_finalizer_type, - rawrefcount_clear_wr) + rawrefcount_clear_wr, + rawrefcount_tuple_maybe_untrack) def _collect(self, major, expected_trigger=0): if major: @@ -138,7 +153,7 @@ return p1, p1ref, check_alive def _rawrefcount_pyobj(self, create_immortal=False, is_gc=True, - tracked=True): + tracked=True, tuple=tuple): r1 = lltype.malloc(PYOBJ_HDR, flavor='raw', immortal=create_immortal) r1.c_ob_refcnt = 0 @@ -146,7 +161,7 @@ r1addr = llmemory.cast_ptr_to_adr(r1) if is_gc: - self._rawrefcount_add_gc(tracked) + self._rawrefcount_add_gc(tracked, tuple) self.pyobjs.append(r1) self.is_pygc.append(is_gc) @@ -161,7 +176,7 @@ def _rawrefcount_pair(self, intval, is_light=False, is_pyobj=False, create_old=False, create_immortal=False, rooted=False, force_external=False, is_gc=True, - tracked=True): + tracked=True, tuple=tuple): if is_light: rc = REFCNT_FROM_PYPY_LIGHT else: @@ -194,7 +209,7 @@ r1addr = llmemory.cast_ptr_to_adr(r1) if is_gc: - self._rawrefcount_add_gc(tracked) + self._rawrefcount_add_gc(tracked, tuple) self.pyobjs.append(r1) self.is_pygc.append(is_gc) @@ -222,16 +237,22 @@ return p1 return p1, p1ref, r1, r1addr, check_alive - def _rawrefcount_add_gc(self, tracked): + def _rawrefcount_add_gc(self, tracked, tuple): r1gc = lltype.malloc(PYOBJ_GC_HDR, flavor='raw', immortal=True) self.gcobjs.append(r1gc) if tracked: r1gc.c_gc_refs = 0 - r1gc.c_gc_next = self.pyobj_list - r1gc.c_gc_prev = self.pyobj_list.c_gc_prev - r1gc.c_gc_prev.c_gc_next = r1gc - self.pyobj_list.c_gc_prev = r1gc + if tuple: + r1gc.c_gc_next = self.tuple_list + r1gc.c_gc_prev = self.tuple_list.c_gc_prev + r1gc.c_gc_prev.c_gc_next = r1gc + self.tuple_list.c_gc_prev = r1gc + else: + r1gc.c_gc_next = self.pyobj_list + r1gc.c_gc_prev = self.pyobj_list.c_gc_prev + r1gc.c_gc_prev.c_gc_next = r1gc + self.pyobj_list.c_gc_prev = r1gc else: r1gc.c_gc_refs = RAWREFCOUNT_REFS_UNTRACKED @@ -485,7 +506,7 @@ class NodeInfo: def __init__(self, type, alive, ext_refcnt, finalizer, resurrect, - delete, garbage): + delete, garbage, tuple): self.type = type self.alive = alive self.ext_refcnt = ext_refcnt @@ -493,6 +514,7 @@ self.resurrect = resurrect self.delete = delete self.garbage = garbage + self.tuple = tuple class WeakrefNode(BorderNode): def __init__(self, p, pref, r, raddr, check_alive, info, r_dest, @@ -529,11 +551,12 @@ resurrect = attr['resurrect'] if 'resurrect' in attr else None delete = attr['delete'] if 'delete' in attr else None garbage = True if 'garbage' in attr else False + tuple = attr['tuple'] == "y" if 'tuple' in attr else False info = NodeInfo(type, alive, ext_refcnt, finalizer, resurrect, - delete, garbage) + delete, garbage, tuple) if type == "C": r, raddr, check_alive = self._rawrefcount_pyobj( - tracked=tracked) + tracked=tracked, tuple=tuple) r.c_ob_refcnt += ext_refcnt nodes[name] = CPythonNode(r, raddr, check_alive, info) elif type == "P": @@ -545,7 +568,8 @@ elif type == "B": p, pref, r, raddr, check_alive =\ self._rawrefcount_pair(42 + i, rooted=rooted, - create_old=True, tracked=tracked) + create_old=True, tracked=tracked, + tuple=tuple) r.c_ob_refcnt += ext_refcnt nodes[name] = BorderNode(p, pref, r, raddr, check_alive, info) i += 1 diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -483,11 +483,13 @@ self.rawrefcount_init_ptr = getfn( GCClass.rawrefcount_init, [s_gc, SomePtr(GCClass.RAWREFCOUNT_DEALLOC_TRIGGER), - SomePtr(GCClass.RAWREFCOUNT_TRAVERSE), SomeAddress(), + SomePtr(GCClass.RAWREFCOUNT_TRAVERSE), + SomeAddress(), SomeAddress(), SomePtr(GCClass.RAWREFCOUNT_GC_AS_PYOBJ), SomePtr(GCClass.RAWREFCOUNT_PYOBJ_AS_GC), SomePtr(GCClass.RAWREFCOUNT_FINALIZER_TYPE), - SomePtr(GCClass.RAWREFCOUNT_CLEAR_WR_TYPE)], + SomePtr(GCClass.RAWREFCOUNT_CLEAR_WR_TYPE), + SomePtr(GCClass.RAWREFCOUNT_MAYBE_UNTRACK_TUPLE)], annmodel.s_None) self.rawrefcount_create_link_pypy_ptr = getfn( GCClass.rawrefcount_create_link_pypy, @@ -1365,8 +1367,8 @@ self.pop_roots(hop, livevars) def gct_gc_rawrefcount_init(self, hop): - [v_fnptr, v_fnptr2, v_pyobj_list, v_fnptr3, v_fnptr4, - v_fnptr5, v_fnptr6] = hop.spaceop.args + [v_fnptr, v_fnptr2, v_pyobj_list, v_tuple_list, v_fnptr3, v_fnptr4, + v_fnptr5, v_fnptr6, v_fnptr7] = hop.spaceop.args assert v_fnptr.concretetype == self.GCClass.RAWREFCOUNT_DEALLOC_TRIGGER assert v_fnptr2.concretetype == self.GCClass.RAWREFCOUNT_TRAVERSE # TODO add assert for v_pyobj_list, improve asserts (types not same but equal) @@ -1374,8 +1376,8 @@ # assert v_fnptr4.concretetype == self.GCClass.RAWREFCOUNT_PYOBJ_AS_GC hop.genop("direct_call", [self.rawrefcount_init_ptr, self.c_const_gc, v_fnptr, - v_fnptr2, v_pyobj_list, v_fnptr3, v_fnptr4, v_fnptr5, - v_fnptr6]) + v_fnptr2, v_pyobj_list, v_tuple_list, v_fnptr3, v_fnptr4, + v_fnptr5, v_fnptr6, v_fnptr7]) def gct_gc_rawrefcount_create_link_pypy(self, hop): [v_gcobj, v_pyobject] = hop.spaceop.args diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py --- a/rpython/rlib/rawrefcount.py +++ b/rpython/rlib/rawrefcount.py @@ -308,8 +308,10 @@ _about_ = init def compute_result_annotation(self, s_dealloc_callback, s_tp_traverse, - s_pyobj_list, s_as_gc, s_as_pyobj, - a_finalizer_type, a_clear_wr): + s_pyobj_list, v_tuple_list, + s_as_gc, s_as_pyobj, + a_finalizer_type, a_clear_wr, + a_maybe_untrack_tuple): from rpython.rtyper.llannotation import SomePtr assert isinstance(s_dealloc_callback, SomePtr) # ll-ptr-to-function assert isinstance(s_tp_traverse, SomePtr) @@ -317,14 +319,18 @@ assert isinstance(s_as_pyobj, SomePtr) assert isinstance(a_finalizer_type, SomePtr) assert isinstance(a_clear_wr, SomePtr) + assert isinstance(a_maybe_untrack_tuple, SomePtr) def specialize_call(self, hop): hop.exception_cannot_occur() - v_dealloc_callback, v_tp_traverse, v_pyobj_list, v_as_gc, \ - v_as_pyobj, v_finalizer_type, v_clear_wr = hop.inputargs(*hop.args_r) + v_dealloc_callback, v_tp_traverse, v_pyobj_list, v_tuple_list, \ + v_as_gc, v_as_pyobj, v_finalizer_type, \ + v_clear_wr, v_maybe_untrack_tuple = hop.inputargs(*hop.args_r) hop.genop('gc_rawrefcount_init', [v_dealloc_callback, v_tp_traverse, - v_pyobj_list, v_as_gc, v_as_pyobj, - v_finalizer_type, v_clear_wr]) + v_pyobj_list, v_tuple_list, + v_as_gc, v_as_pyobj, + v_finalizer_type, v_clear_wr, + v_maybe_untrack_tuple]) class Entry(ExtRegistryEntry): From pypy.commits at gmail.com Sat Jun 22 08:16:35 2019 From: pypy.commits at gmail.com (fijal) Date: Sat, 22 Jun 2019 05:16:35 -0700 (PDT) Subject: [pypy-commit] pypy arm64: finish cond_call Message-ID: <5d0e1c23.1c69fb81.6d72c.882e@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96838:eaa56e09349e Date: 2019-06-22 11:12 +0000 http://bitbucket.org/pypy/pypy/changeset/eaa56e09349e/ Log: finish cond_call diff --git a/rpython/jit/backend/aarch64/TODO b/rpython/jit/backend/aarch64/TODO --- a/rpython/jit/backend/aarch64/TODO +++ b/rpython/jit/backend/aarch64/TODO @@ -1,4 +1,4 @@ * int_add - IMM * int_cmp - IMM * guard_nonnull_class - think about a better way -* cond_call and following guard_exception \ No newline at end of file +* cond_call and following guard_exception diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -793,7 +793,8 @@ if arglocs is not None: asm_guard_operations[guard_num](self, guard_op, fcond, arglocs) regalloc.next_instruction() # advance one more - regalloc.possibly_free_vars(guard_op.getfailargs()) + if guard_op.is_guard(): # can be also cond_call + regalloc.possibly_free_vars(guard_op.getfailargs()) regalloc.possibly_free_vars_for_op(guard_op) else: arglocs = regalloc_operations[opnum](regalloc, op) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -596,7 +596,7 @@ self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs, array=True) - def emit_op_cond_call(self, op, arglocs): + def _emit_op_cond_call(self, op, arglocs, fcond): #if len(arglocs) == 2: # res_loc = arglocs[1] # cond_call_value #else: @@ -604,7 +604,8 @@ # see x86.regalloc for why we skip res_loc in the gcmap res_loc = None - self.mc.CMP_ri(arglocs[0].value, 0) + if arglocs[0] is not None: # otherwise result already in CC + self.mc.CMP_ri(arglocs[0].value, 0) gcmap = self._regalloc.get_gcmap([res_loc]) @@ -637,7 +638,16 @@ # self.pop_gcmap(self.mc) pmc = OverwritingBuilder(self.mc, jmp_adr, WORD) - pmc.B_ofs_cond(self.mc.currpos() - jmp_adr, c.EQ) + pmc.B_ofs_cond(self.mc.currpos() - jmp_adr, fcond) + # might be overridden again to skip over the following + # guard_no_exception too + self.previous_cond_call_jcond = jmp_adr, fcond + + def emit_op_cond_call(self, op, arglocs): + self._emit_op_cond_call(op, arglocs, c.EQ) + + def emit_guard_op_cond_call(self, op, fcond, arglocs): + self._emit_op_cond_call(op, arglocs, c.get_opposite_of(fcond)) def _write_barrier_fastpath(self, mc, descr, arglocs, array=False, is_frame=False): # Write code equivalent to write_barrier() in the GC: it checks diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -674,7 +674,7 @@ self._compute_hint_frame_locations_from_descr(descr) return [] - def prepare_op_cond_call(self, op): + def _prepare_op_cond_call(self, op, res_in_cc): assert 2 <= op.numargs() <= 4 + 2 v = op.getarg(1) assert isinstance(v, Const) @@ -684,7 +684,10 @@ arg = op.getarg(i) self.make_sure_var_in_reg(arg, args_so_far, selected_reg=reg) args_so_far.append(arg) - argloc = self.make_sure_var_in_reg(op.getarg(0), args_so_far) + if res_in_cc: + argloc = None + else: + argloc = self.make_sure_var_in_reg(op.getarg(0), args_so_far) if op.type == 'v': # a plain COND_CALL. Calls the function when args[0] is @@ -710,6 +713,13 @@ self.assembler.guard_success_cc = c.EQ return [tmpreg, resloc] + def prepare_op_cond_call(self, op): + return self._prepare_op_cond_call(op, False) + + def prepare_guard_op_cond_call(self, op, prevop): + fcond = self.assembler.dispatch_comparison(prevop) + locs = self._prepare_op_cond_call(op, True) + return locs, fcond def prepare_op_finish(self, op): # the frame is in fp, but we have to point where in the frame is From pypy.commits at gmail.com Sat Jun 22 08:16:37 2019 From: pypy.commits at gmail.com (fijal) Date: Sat, 22 Jun 2019 05:16:37 -0700 (PDT) Subject: [pypy-commit] pypy arm64: cond_call_value_i Message-ID: <5d0e1c25.1c69fb81.47eab.506b@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96839:ebaaa5860fff Date: 2019-06-22 11:39 +0000 http://bitbucket.org/pypy/pypy/changeset/ebaaa5860fff/ Log: cond_call_value_i diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -597,12 +597,11 @@ array=True) def _emit_op_cond_call(self, op, arglocs, fcond): - #if len(arglocs) == 2: - # res_loc = arglocs[1] # cond_call_value - #else: - # res_loc = None # cond_call + if len(arglocs) == 2: + res_loc = arglocs[1] # cond_call_value + else: + res_loc = None # cond_call # see x86.regalloc for why we skip res_loc in the gcmap - res_loc = None if arglocs[0] is not None: # otherwise result already in CC self.mc.CMP_ri(arglocs[0].value, 0) @@ -631,8 +630,7 @@ self.mc.BL(cond_call_adr) # if this is a COND_CALL_VALUE, we need to move the result in place - # from its current location (which is, unusually, in r4: see - # cond_call_slowpath) + # from its current location if res_loc is not None: self.mc.MOV_rr(res_loc.value, r.ip1.value) # @@ -646,6 +644,9 @@ def emit_op_cond_call(self, op, arglocs): self._emit_op_cond_call(op, arglocs, c.EQ) + def emit_op_cond_call_value_i(self, op, arglocs): + self._emit_op_cond_call(op, arglocs, c.NE) + def emit_guard_op_cond_call(self, op, fcond, arglocs): self._emit_op_cond_call(op, arglocs, c.get_opposite_of(fcond)) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -694,7 +694,6 @@ # true. Often used just after a comparison operation. return [argloc] else: - XXX # COND_CALL_VALUE_I/R. Calls the function when args[0] # is equal to 0 or NULL. Returns the result from the # function call if done, or args[0] if it was not 0/NULL. @@ -708,14 +707,14 @@ args = op.getarglist() resloc = self.rm.force_result_in_reg(op, args[0], forbidden_vars=args[2:]) - # Test the register for the result. - self.assembler.mc.CMP_ri(resloc.value, 0) - self.assembler.guard_success_cc = c.EQ - return [tmpreg, resloc] + return [argloc, resloc] def prepare_op_cond_call(self, op): return self._prepare_op_cond_call(op, False) + def prepare_op_cond_call_value_i(self, op): + return self._prepare_op_cond_call(op, False) + def prepare_guard_op_cond_call(self, op, prevop): fcond = self.assembler.dispatch_comparison(prevop) locs = self._prepare_op_cond_call(op, True) From pypy.commits at gmail.com Sat Jun 22 08:16:39 2019 From: pypy.commits at gmail.com (fijal) Date: Sat, 22 Jun 2019 05:16:39 -0700 (PDT) Subject: [pypy-commit] pypy arm64: call_may_force Message-ID: <5d0e1c27.1c69fb81.afcdf.0af2@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96840:5142be166d5c Date: 2019-06-22 12:08 +0000 http://bitbucket.org/pypy/pypy/changeset/5142be166d5c/ Log: call_may_force diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -791,11 +791,22 @@ guard_num = guard_op.getopnum() arglocs, fcond = guard_operations[guard_num](regalloc, guard_op, op) if arglocs is not None: - asm_guard_operations[guard_num](self, guard_op, fcond, arglocs) + asm_guard_operations[guard_num](self, op, guard_op, fcond, arglocs) regalloc.next_instruction() # advance one more if guard_op.is_guard(): # can be also cond_call regalloc.possibly_free_vars(guard_op.getfailargs()) regalloc.possibly_free_vars_for_op(guard_op) + elif rop.is_call_may_force(op.getopnum()): + guard_op = operations[i + 1] # has to exist + guard_num = guard_op.getopnum() + assert guard_num in (rop.GUARD_NOT_FORCED, rop.GUARD_NOT_FORCED_2) + arglocs, fcond = guard_operations[guard_num](regalloc, guard_op, op) + if arglocs is not None: + asm_guard_operations[guard_num](self, op, guard_op, fcond, arglocs) + # fcond is abused here to pass the number of args + regalloc.next_instruction() # advance one more + regalloc.possibly_free_vars(guard_op.getfailargs()) + regalloc.possibly_free_vars_for_op(guard_op) else: arglocs = regalloc_operations[opnum](regalloc, op) if arglocs is not None: diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -481,11 +481,11 @@ else: self.mc.BRK() - def emit_guard_op_guard_true(self, guard_op, fcond, arglocs): + def emit_guard_op_guard_true(self, op, guard_op, fcond, arglocs): self._emit_guard(guard_op, fcond, arglocs) emit_guard_op_guard_no_overflow = emit_guard_op_guard_true - def emit_guard_op_guard_false(self, guard_op, fcond, arglocs): + def emit_guard_op_guard_false(self, op, guard_op, fcond, arglocs): self._emit_guard(guard_op, c.get_opposite_of(fcond), arglocs) emit_guard_op_guard_overflow = emit_guard_op_guard_false @@ -647,7 +647,7 @@ def emit_op_cond_call_value_i(self, op, arglocs): self._emit_op_cond_call(op, arglocs, c.NE) - def emit_guard_op_cond_call(self, op, fcond, arglocs): + def emit_guard_op_cond_call(self, prevop, op, fcond, arglocs): self._emit_op_cond_call(op, arglocs, c.get_opposite_of(fcond)) def _write_barrier_fastpath(self, mc, descr, arglocs, array=False, is_frame=False): @@ -804,9 +804,31 @@ else: cb.emit_no_collect() + def emit_guard_op_guard_not_forced(self, op, guard_op, fcond, arglocs): + # arglocs is call locs + guard_locs, split them + assert fcond == op.numargs() + 3 + call_args = arglocs[:fcond] + guard_locs = arglocs[fcond:] + self._store_force_index(guard_op) + self._emit_call(op, call_args) + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + self.mc.LDR_ri(r.ip0.value, r.fp.value, ofs) + self.mc.CMP_ri(r.ip0.value, 0) + self._emit_guard(guard_op, c.EQ, guard_locs) + + def _store_force_index(self, guard_op): + faildescr = guard_op.getdescr() + faildescrindex = self.get_gcref_from_faildescr(faildescr) + ofs = self.cpu.get_ofs_of_frame_field('jf_force_descr') + self.load_from_gc_table(r.ip0.value, faildescrindex) + self.store_reg(self.mc, r.ip0, r.fp, ofs) + def emit_op_label(self, op, arglocs): pass + def emit_op_force_token(self, op, arglocs): + self.mc.MOV_rr(arglocs[0].value, r.fp.value) + def emit_op_jump(self, op, arglocs): target_token = op.getdescr() assert isinstance(target_token, TargetToken) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -634,6 +634,11 @@ else: return self.rm.after_call(v) + def prepare_guard_op_guard_not_forced(self, op, prev_op): + arglocs = self._prepare_call(prev_op, save_all_regs=True) + guard_locs = self._guard_impl(op) + return arglocs + guard_locs, len(arglocs) + def prepare_op_label(self, op): descr = op.getdescr() assert isinstance(descr, TargetToken) @@ -720,6 +725,11 @@ locs = self._prepare_op_cond_call(op, True) return locs, fcond + def prepare_op_force_token(self, op): + # XXX regular reg + res_loc = self.force_allocate_reg(op) + return [res_loc] + def prepare_op_finish(self, op): # the frame is in fp, but we have to point where in the frame is # the potential argument to FINISH diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2506,6 +2506,7 @@ def test_force_operations_returning_void(self): values = [] def maybe_force(token, flag): + print "CALLED WITH " + str(flag) if flag: deadframe = self.cpu.force(token) values.append(self.cpu.get_latest_descr(deadframe)) From pypy.commits at gmail.com Sat Jun 22 08:16:40 2019 From: pypy.commits at gmail.com (fijal) Date: Sat, 22 Jun 2019 05:16:40 -0700 (PDT) Subject: [pypy-commit] pypy arm64: guard_not_forced_2 Message-ID: <5d0e1c28.1c69fb81.84973.e77d@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96841:31ae58bae243 Date: 2019-06-22 12:15 +0000 http://bitbucket.org/pypy/pypy/changeset/31ae58bae243/ Log: guard_not_forced_2 diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -1,5 +1,6 @@ from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.rarithmetic import r_uint from rpython.rtyper.lltypesystem import rffi, lltype from rpython.jit.metainterp.history import (AbstractFailDescr, ConstInt, INT, FLOAT, REF) @@ -823,12 +824,22 @@ self.load_from_gc_table(r.ip0.value, faildescrindex) self.store_reg(self.mc, r.ip0, r.fp, ofs) - def emit_op_label(self, op, arglocs): - pass + def emit_op_guard_not_forced_2(self, op, arglocs): + self.store_force_descr(op, arglocs[1:], arglocs[0].value) + + def store_force_descr(self, op, fail_locs, frame_depth): + pos = self.mc.currpos() + guard_token = self.build_guard_token(op, frame_depth, fail_locs, pos, c.AL) + self._finish_gcmap = guard_token.gcmap + self._store_force_index(op) + self.store_info_on_descr(pos, guard_token) def emit_op_force_token(self, op, arglocs): self.mc.MOV_rr(arglocs[0].value, r.fp.value) + def emit_op_label(self, op, arglocs): + pass + def emit_op_jump(self, op, arglocs): target_token = op.getdescr() assert isinstance(target_token, TargetToken) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -639,6 +639,12 @@ guard_locs = self._guard_impl(op) return arglocs + guard_locs, len(arglocs) + def prepare_op_guard_not_forced_2(self, op): + self.rm.before_call(op.getfailargs(), save_all_regs=True) + self.vfprm.before_call(op.getfailargs(), save_all_regs=True) + fail_locs = self._guard_impl(op) + return fail_locs + def prepare_op_label(self, op): descr = op.getdescr() assert isinstance(descr, TargetToken) From pypy.commits at gmail.com Sat Jun 22 11:45:01 2019 From: pypy.commits at gmail.com (fijal) Date: Sat, 22 Jun 2019 08:45:01 -0700 (PDT) Subject: [pypy-commit] pypy arm64: add missing files and have some basic progress Message-ID: <5d0e4cfd.1c69fb81.2bf47.87e9@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96842:ea0ee0f66afd Date: 2019-06-22 15:44 +0000 http://bitbucket.org/pypy/pypy/changeset/ea0ee0f66afd/ Log: add missing files and have some basic progress diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -624,7 +624,8 @@ self.mc.BL(target) return startpos - def push_gcmap(self, mc, gcmap): + def push_gcmap(self, mc, gcmap, store=True): + assert store ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') ptr = rffi.cast(lltype.Signed, gcmap) mc.gen_load_int(r.ip0.value, ptr) @@ -796,7 +797,8 @@ if guard_op.is_guard(): # can be also cond_call regalloc.possibly_free_vars(guard_op.getfailargs()) regalloc.possibly_free_vars_for_op(guard_op) - elif rop.is_call_may_force(op.getopnum()): + elif (rop.is_call_may_force(op.getopnum()) or + rop.is_call_release_gil(op.getopnum())): guard_op = operations[i + 1] # has to exist guard_num = guard_op.getopnum() assert guard_num in (rop.GUARD_NOT_FORCED, rop.GUARD_NOT_FORCED_2) diff --git a/rpython/jit/backend/aarch64/callbuilder.py b/rpython/jit/backend/aarch64/callbuilder.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/callbuilder.py @@ -0,0 +1,163 @@ + +from rpython.jit.backend.llsupport.callbuilder import AbstractCallBuilder +from rpython.jit.backend.aarch64.arch import WORD +from rpython.jit.metainterp.history import INT, FLOAT, REF +from rpython.jit.backend.aarch64 import registers as r +from rpython.jit.backend.aarch64.jump import remap_frame_layout # we use arm algo + +from rpython.rlib.objectmodel import we_are_translated +from rpython.rtyper.lltypesystem import rffi + +class Aarch64CallBuilder(AbstractCallBuilder): + def __init__(self, assembler, fnloc, arglocs, + resloc=r.x0, restype=INT, ressize=WORD, ressigned=True): + AbstractCallBuilder.__init__(self, assembler, fnloc, arglocs, + resloc, restype, ressize) + self.current_sp = 0 + + def prepare_arguments(self): + arglocs = self.arglocs + non_float_locs = [] + non_float_regs = [] + float_locs = [] + float_regs = [] + stack_locs = [] + free_regs = [r.x7, r.x6, r.x5, r.x4, r.x3, r.x2, r.x1, r.x0] + free_float_regs = [r.d7, r.d6, r.d5, r.d4, r.d3, r.d2, r.d1, r.d0] + for arg in arglocs: + if arg.type == FLOAT: + if free_float_regs: + float_locs.append(arg) + float_regs.append(free_float_regs.pop()) + else: + stack_locs.append(arg) + else: + if free_regs: + non_float_locs.append(arg) + non_float_regs.append(free_regs.pop()) + else: + stack_locs.append(arg) + remap_frame_layout(self.asm, non_float_locs, non_float_regs, r.ip0) + if float_locs: + remap_frame_layout(self.asm, float_locs, float_regs, r.d8) + # move the remaining things to stack and adjust the stack + if not stack_locs: + return + adj = len(stack_locs) + (len(stack_locs) & 1) + self.mc.SUB_ri(r.sp.value, r.sp.value, adj * WORD) + self.current_sp = adj + c = 0 + for loc in stack_locs: + self.asm.mov_loc_to_raw_stack(loc, c) + c += WORD + + def push_gcmap(self): + noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack() + gcmap = self.asm._regalloc.get_gcmap([r.x0], noregs=noregs) + self.asm.push_gcmap(self.mc, gcmap) + + def pop_gcmap(self): + self.asm._reload_frame_if_necessary(self.mc) + self.asm.pop_gcmap(self.mc) + + def emit_raw_call(self): + #the actual call + if self.fnloc.is_imm(): + self.mc.BL(self.fnloc.value) + return + if self.fnloc.is_stack(): + self.mc.LDR_ri(r.ip0.value, r.fp.value, self.fnloc.value) + self.mc.BLR_r(r.ip0.value) + else: + assert self.fnloc.is_core_reg() + self.mc.BLR_r(self.fnloc.value) + + def restore_stack_pointer(self): + assert self.current_sp & 1 == 0 # always adjusted to 16 bytes + if self.current_sp == 0: + return + self.mc.ADD_ri(r.sp.value, r.sp.value, self.current_sp * WORD) + self.current_sp = 0 + + def load_result(self): + resloc = self.resloc + if self.restype == 'S': + XXX + self.mc.VMOV_sc(resloc.value, r.s0.value) + elif self.restype == 'L': + YYY + assert resloc.is_vfp_reg() + self.mc.FMDRR(resloc.value, r.r0.value, r.r1.value) + # ensure the result is wellformed and stored in the correct location + if resloc is not None and resloc.is_core_reg(): + self._ensure_result_bit_extension(resloc, + self.ressize, self.ressign) + + def _ensure_result_bit_extension(self, resloc, size, signed): + if size == WORD: + return + if size == 4: + if not signed: # unsigned int + self.mc.LSL_ri(resloc.value, resloc.value, 32) + self.mc.LSR_ri(resloc.value, resloc.value, 32) + else: # signed int + self.mc.LSL_ri(resloc.value, resloc.value, 32) + self.mc.ASR_ri(resloc.value, resloc.value, 32) + elif size == 2: + if not signed: + self.mc.LSL_ri(resloc.value, resloc.value, 48) + self.mc.LSR_ri(resloc.value, resloc.value, 48) + else: + self.mc.LSL_ri(resloc.value, resloc.value, 48) + self.mc.ASR_ri(resloc.value, resloc.value, 48) + elif size == 1: + if not signed: # unsigned char + self.mc.AND_ri(resloc.value, resloc.value, 0xFF) + else: + self.mc.LSL_ri(resloc.value, resloc.value, 56) + self.mc.ASR_ri(resloc.value, resloc.value, 56) + + def call_releasegil_addr_and_move_real_arguments(self, fastgil): + assert self.is_call_release_gil + assert not self.asm._is_asmgcc() + + # Save this thread's shadowstack pointer into r7, for later comparison + gcrootmap = self.asm.cpu.gc_ll_descr.gcrootmap + if gcrootmap: + XXX + rst = gcrootmap.get_root_stack_top_addr() + self.mc.gen_load_int(r.r5.value, rst) + self.mc.LDR_ri(r.r7.value, r.r5.value) + + # change 'rpy_fastgil' to 0 (it should be non-zero right now) + self.mc.DMB() + self.mc.gen_load_int(r.ip1.value, fastgil) + self.mc.MOVZ_r_u16(r.ip0.value, 0, 0) + self.mc.STR_ri(r.ip0.value, r.ip1.value, 0) + + if not we_are_translated(): # for testing: we should not access + self.mc.ADD_ri(r.fp.value, r.fp.value, 1) # fp any more + + def write_real_errno(self, save_err): + if save_err & rffi.RFFI_READSAVED_ERRNO: + xxx + elif save_err & rffi.RFFI_ZERO_ERRNO_BEFORE: + yyy + + def read_real_errno(self, save_err): + if save_err & rffi.RFFI_SAVE_ERRNO: + xxx + + def move_real_result_and_call_reacqgil_addr(self, fastgil): + xxx + + def get_result_locs(self): + if self.resloc is None: + return [], [] + if self.resloc.is_vfp_reg(): + if self.restype == 'L': # long long + return [r.r0, r.r1], [] + else: + return [], [r.d0] + assert self.resloc.is_core_reg() + return [r.r0], [] diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -372,6 +372,9 @@ def BRK(self): self.write32(0b11010100001 << 21) + def DMB(self): + self.write32(0b1101010100000011001111110111111) + def gen_load_int_full(self, r, value): self.MOVZ_r_u16(r, value & 0xFFFF, 0) self.MOVK_r_u16(r, (value >> 16) & 0xFFFF, 16) diff --git a/rpython/jit/backend/aarch64/jump.py b/rpython/jit/backend/aarch64/jump.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/jump.py @@ -0,0 +1,113 @@ + +from rpython.jit.backend.aarch64 import registers as r + +def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): + pending_dests = len(dst_locations) + srccount = {} # maps dst_locations to how many times the same + # location appears in src_locations + for dst in dst_locations: + key = dst.as_key() + assert key not in srccount, "duplicate value in dst_locations!" + srccount[key] = 0 + for i in range(len(dst_locations)): + src = src_locations[i] + if src.is_imm(): + continue + key = src.as_key() + if key in srccount: + if key == dst_locations[i].as_key(): + # ignore a move "x = x" + # setting any "large enough" negative value is ok, but + # be careful of overflows, don't use -sys.maxint + srccount[key] = -len(dst_locations) - 1 + pending_dests -= 1 + else: + srccount[key] += 1 + + while pending_dests > 0: + progress = False + for i in range(len(dst_locations)): + dst = dst_locations[i] + key = dst.as_key() + if srccount[key] == 0: + srccount[key] = -1 # means "it's done" + pending_dests -= 1 + src = src_locations[i] + if not src.is_imm(): + key = src.as_key() + if key in srccount: + srccount[key] -= 1 + _move(assembler, src, dst, tmpreg) + progress = True + if not progress: + # we are left with only pure disjoint cycles + sources = {} # maps dst_locations to src_locations + for i in range(len(dst_locations)): + src = src_locations[i] + dst = dst_locations[i] + sources[dst.as_key()] = src + # + for i in range(len(dst_locations)): + dst = dst_locations[i] + originalkey = dst.as_key() + if srccount[originalkey] >= 0: + assembler.push_locations([dst]) + while True: + key = dst.as_key() + assert srccount[key] == 1 + # ^^^ because we are in a simple cycle + srccount[key] = -1 + pending_dests -= 1 + src = sources[key] + if src.as_key() == originalkey: + break + _move(assembler, src, dst, tmpreg) + dst = src + assembler.pop_locations([dst]) + assert pending_dests == 0 + + +def _move(assembler, src, dst, tmpreg): + if dst.is_stack() and src.is_stack(): + assembler.regalloc_mov(src, tmpreg) + src = tmpreg + assembler.regalloc_mov(src, dst) + + +def remap_frame_layout_mixed(assembler, + src_locations1, dst_locations1, tmpreg1, + src_locations2, dst_locations2, tmpreg2): + # find and push the xmm stack locations from src_locations2 that + # are going to be overwritten by dst_locations1 + extrapushes = [] + extrapops = [] + dst_keys = {} + for loc in dst_locations1: + dst_keys[loc.as_key()] = None + src_locations2red = [] + dst_locations2red = [] + for i in range(len(src_locations2)): + loc = src_locations2[i] + dstloc = dst_locations2[i] + if loc.is_stack(): + key = loc.as_key() + if key in dst_keys: + extrapushes.append(loc) + extrapops.append(dstloc) + continue + src_locations2red.append(loc) + dst_locations2red.append(dstloc) + src_locations2 = src_locations2red + dst_locations2 = dst_locations2red + + assembler.push_locations(extrapushes) + + # + # remap the integer and pointer registers and stack locations + remap_frame_layout(assembler, src_locations1, dst_locations1, tmpreg1) + # + # remap the vfp registers and stack locations + remap_frame_layout(assembler, src_locations2, dst_locations2, tmpreg2) + # + # finally, pop the extra xmm stack locations + assembler.pop_locations(extrapops) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -774,7 +774,8 @@ emit_op_call_f = _genop_call emit_op_call_n = _genop_call - def _emit_call(self, op, arglocs, is_call_release_gil=False): + def _emit_call(self, op, arglocs): + is_call_release_gil = rop.is_call_release_gil(op.getopnum()) # args = [resloc, size, sign, args...] from rpython.jit.backend.llsupport.descr import CallDescr diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -311,6 +311,8 @@ return [] prepare_op_jit_debug = void + prepare_op_enter_portal_frame = void + prepare_op_leave_portal_frame = void def prepare_int_ri(self, op, res_in_cc): boxes = op.getarglist() @@ -635,7 +637,11 @@ return self.rm.after_call(v) def prepare_guard_op_guard_not_forced(self, op, prev_op): - arglocs = self._prepare_call(prev_op, save_all_regs=True) + if rop.is_call_release_gil(prev_op.getopnum()): + arglocs = self._prepare_call(prev_op, save_all_regs=True, + first_arg_index=2) + else: + arglocs = self._prepare_call(prev_op, save_all_regs=True) guard_locs = self._guard_impl(op) return arglocs + guard_locs, len(arglocs) diff --git a/rpython/jit/backend/aarch64/test/test_basic.py b/rpython/jit/backend/aarch64/test/test_basic.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/test/test_basic.py @@ -0,0 +1,42 @@ +import py +from rpython.jit.metainterp.test import test_ajit +from rpython.rlib.jit import JitDriver +from rpython.jit.metainterp.test.support import LLJitMixin +from rpython.jit.backend.detect_cpu import getcpuclass + +class JitAarch64Mixin(LLJitMixin): + CPUClass = getcpuclass() + # we have to disable unroll + enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap" + basic = False + + def check_jumps(self, maxcount): + pass + +class TestBasic(JitAarch64Mixin, test_ajit.BaseLLtypeTests): + # for the individual tests see + # ====> ../../../metainterp/test/test_ajit.py + def test_bug(self): + jitdriver = JitDriver(greens = [], reds = ['n']) + class X(object): + pass + def f(n): + while n > -100: + jitdriver.can_enter_jit(n=n) + jitdriver.jit_merge_point(n=n) + x = X() + x.arg = 5 + if n <= 0: break + n -= x.arg + x.arg = 6 # prevents 'x.arg' from being annotated as constant + return n + res = self.meta_interp(f, [31], enable_opts='') + assert res == -4 + + def test_r_dict(self): + # a Struct that belongs to the hash table is not seen as being + # included in the larger Array + py.test.skip("issue with ll2ctypes") + + def test_free_object(self): + py.test.skip("issue of freeing, probably with ll2ctypes") From pypy.commits at gmail.com Sat Jun 22 18:20:42 2019 From: pypy.commits at gmail.com (mattip) Date: Sat, 22 Jun 2019 15:20:42 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: fix translation Message-ID: <5d0ea9ba.1c69fb81.1b571.91f5@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96843:6ed43dc686fc Date: 2019-06-23 01:15 +0300 http://bitbucket.org/pypy/pypy/changeset/6ed43dc686fc/ Log: fix translation diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -19,8 +19,10 @@ MEMORYVIEW_PIL = 0x0010 def is_multiindex(space, w_key): + from pypy.objspace.std.tupleobject import W_AbstractTupleObject if not space.isinstance_w(w_key, space.w_tuple): return 0 + assert isinstance(w_key, W_AbstractTupleObject) length = space.len_w(w_key) i = 0 while i < length: @@ -31,8 +33,10 @@ return 1 def is_multislice(space, w_key): + from pypy.objspace.std.tupleobject import W_AbstractTupleObject if not space.isinstance_w(w_key, space.w_tuple): return 0 + assert isinstance(w_key, W_AbstractTupleObject) length = space.len_w(w_key) if length == 0: return 0 From pypy.commits at gmail.com Sun Jun 23 01:40:03 2019 From: pypy.commits at gmail.com (mattip) Date: Sat, 22 Jun 2019 22:40:03 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: make memroyview.setitem failure logic more like cpython Message-ID: <5d0f10b3.1c69fb81.c4a9e.fc92@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96844:4cc3da9ff28d Date: 2019-06-23 08:39 +0300 http://bitbucket.org/pypy/pypy/changeset/4cc3da9ff28d/ Log: make memroyview.setitem failure logic more like cpython diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -174,6 +174,9 @@ "cannot index %d-dimension view with %d-element tuple", length, ndim) + elif is_multislice(space, w_index): + raise oefmt(space.w_NotImplementedError, + "multi-dimensional slicing is not implemented") start = self._start_from_tuple(space, w_index) itemsize = self.getitemsize() val = self.view.bytes_from_value(space, w_obj) diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -299,7 +299,9 @@ raises(TypeError, m.__setitem__, (2, 3), bytearray(b'12')) # slices in 2d memoryviews are not supported at all raises(TypeError, m.__getitem__, (slice(None), 3)) - raises(NotImplementedError, m.__getitem__, (slice(None),)) + raises(TypeError, m.__setitem__, (slice(None), 3), ord('a')) + raises(NotImplementedError, m.__getitem__, (slice(0,1,1), slice(0,1,2))) + raises(NotImplementedError, m.__setitem__, (slice(0,1,1), slice(0,1,2)), ord('a')) class AppTestCtypes(object): spaceconfig = dict(usemodules=['sys', '_rawffi']) From pypy.commits at gmail.com Mon Jun 24 00:04:34 2019 From: pypy.commits at gmail.com (mattip) Date: Sun, 23 Jun 2019 21:04:34 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: remove python2-only slot functions, fix test Message-ID: <5d104bd2.1c69fb81.c0e52.c6e7@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96845:2ee8740e5142 Date: 2019-06-24 07:03 +0300 http://bitbucket.org/pypy/pypy/changeset/2ee8740e5142/ Log: remove python2-only slot functions, fix test diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -30,7 +30,7 @@ from rpython.flowspace.specialcase import register_flow_sc from pypy.module.sys.version import CPYTHON_VERSION -PY3 = CPYTHON_VERSION[0] == 3 +PY2 = CPYTHON_VERSION[0] == 2 # XXX: Also defined in object.h Py_LT = 0 @@ -849,7 +849,7 @@ missing_builtin_slots = [ 'tp_print', 'tp_compare', 'tp_getattr', 'tp_setattr', 'tp_setattro', 'tp_finalize', - 'tp_richcompare', 'tp_del', 'tp_as_buffer.c_bf_getwritebuffer', + 'tp_richcompare', 'tp_del', 'tp_as_number.c_nb_bool', 'tp_as_number.c_nb_coerce', 'tp_as_number.c_nb_inplace_add', 'tp_as_number.c_nb_inplace_subtract', 'tp_as_number.c_nb_inplace_multiply', 'tp_as_number.c_nb_inplace_divide', @@ -863,7 +863,11 @@ 'tp_as_number.c_nb_inplace_matrix_multiply', 'tp_as_sequence.c_sq_slice', 'tp_as_sequence.c_sq_ass_slice', 'tp_as_sequence.c_sq_contains', - 'tp_as_buffer.c_bf_getreadbuffer', + ] +if PY2: + missing_builtin_slots += [ + 'tp_as_buffer.c_bf_getreadbuffer', + 'tp_as_buffer.c_bf_getwritebuffer', ] for name in missing_builtin_slots: slot_factory(name)(make_missing_slot) @@ -1151,7 +1155,7 @@ TPSLOT("__buffer__", "tp_as_buffer.c_bf_getbuffer", None, "wrap_getbuffer", ""), ) -if not PY3: +if PY2: slotdefs += ( TPSLOT("__rbuffer__", "tp_as_buffer.c_bf_getreadbuffer", None, "wrap_getreadbuffer", ""), TPSLOT("__wbuffer__", "tp_as_buffer.c_bf_getwritebuffer", None, "wrap_getwritebuffer", ""), diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -49,8 +49,7 @@ py_obj = make_ref(space, w_obj) assert py_obj.c_ob_type.c_tp_as_buffer assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getbuffer - assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getreadbuffer - assert py_obj.c_ob_type.c_tp_as_buffer.c_bf_getwritebuffer + assert not py_obj.c_ob_type.c_tp_as_buffer.c_bf_releasebuffer class AppTestPyBuffer_FillInfo(AppTestCpythonExtensionBase): From pypy.commits at gmail.com Mon Jun 24 04:06:54 2019 From: pypy.commits at gmail.com (fijal) Date: Mon, 24 Jun 2019 01:06:54 -0700 (PDT) Subject: [pypy-commit] pypy arm64: call_assembler Message-ID: <5d10849e.1c69fb81.8e91a.254d@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96846:ca09029d532f Date: 2019-06-22 18:43 +0000 http://bitbucket.org/pypy/pypy/changeset/ca09029d532f/ Log: call_assembler diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -737,16 +737,16 @@ self.mc.BL(self.stack_check_slowpath, c=c.HI) # call if ip > lr def _call_header(self): - stack_size = (len(r.callee_saved_registers) + 2) * WORD + stack_size = (len(r.callee_saved_registers) + 4) * WORD self.mc.STP_rr_preindex(r.fp.value, r.lr.value, r.sp.value, -stack_size) for i in range(0, len(r.callee_saved_registers), 2): self.mc.STP_rri(r.callee_saved_registers[i].value, r.callee_saved_registers[i + 1].value, r.sp.value, - (i + 2) * WORD) + (i + 4) * WORD) - #self.saved_threadlocal_addr = 0 # at offset 0 from location 'sp' - # ^^^XXX save it from register x1 into some place + self.saved_threadlocal_addr = 3 * WORD # at offset 3 from location 'sp' + self.mc.STR_ri(r.x1.value, r.sp.value, 3 * WORD) # set fp to point to the JITFRAME, passed in argument 'x0' self.mc.MOV_rr(r.fp.value, r.x0.value) @@ -798,7 +798,8 @@ regalloc.possibly_free_vars(guard_op.getfailargs()) regalloc.possibly_free_vars_for_op(guard_op) elif (rop.is_call_may_force(op.getopnum()) or - rop.is_call_release_gil(op.getopnum())): + rop.is_call_release_gil(op.getopnum()) or + rop.is_call_assembler(op.getopnum())): guard_op = operations[i + 1] # has to exist guard_num = guard_op.getopnum() assert guard_num in (rop.GUARD_NOT_FORCED, rop.GUARD_NOT_FORCED_2) @@ -972,13 +973,13 @@ # pop all callee saved registers - stack_size = len(r.callee_saved_registers) * WORD + stack_size = (len(r.callee_saved_registers) + 4) * WORD for i in range(0, len(r.callee_saved_registers), 2): mc.LDP_rri(r.callee_saved_registers[i].value, r.callee_saved_registers[i + 1].value, r.sp.value, - (i + 2) * WORD) + (i + 4) * WORD) mc.LDP_rr_postindex(r.fp.value, r.lr.value, r.sp.value, stack_size) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -3,13 +3,14 @@ from rpython.rlib.rarithmetic import r_uint from rpython.rtyper.lltypesystem import rffi, lltype from rpython.jit.metainterp.history import (AbstractFailDescr, ConstInt, - INT, FLOAT, REF) + INT, FLOAT, REF, VOID) from rpython.jit.backend.aarch64 import registers as r from rpython.jit.backend.aarch64.codebuilder import OverwritingBuilder from rpython.jit.backend.aarch64.callbuilder import Aarch64CallBuilder from rpython.jit.backend.arm import conditions as c, shift +from rpython.jit.backend.aarch64.regalloc import check_imm_arg from rpython.jit.backend.aarch64.arch import JITFRAME_FIXED_SIZE, WORD -from rpython.jit.backend.aarch64.locations import imm +from rpython.jit.backend.aarch64 import locations from rpython.jit.backend.llsupport.assembler import GuardToken, BaseAssembler from rpython.jit.backend.llsupport.gcmap import allocate_gcmap from rpython.jit.backend.llsupport.regalloc import get_scale @@ -42,6 +43,9 @@ return emit_op class ResOpAssembler(BaseAssembler): + def imm(self, v): + return locations.imm(v) + def int_sub_impl(self, op, arglocs, flags=0): l0, l1, res = arglocs if flags: @@ -806,18 +810,98 @@ else: cb.emit_no_collect() + def simple_call(self, fnloc, arglocs, result_loc=r.x0): + if result_loc is None: + result_type = VOID + result_size = 0 + elif result_loc.is_vfp_reg(): + result_type = FLOAT + result_size = WORD + else: + result_type = INT + result_size = WORD + cb = Aarch64CallBuilder(self, fnloc, arglocs, + result_loc, result_type, + result_size) + cb.emit() + def emit_guard_op_guard_not_forced(self, op, guard_op, fcond, arglocs): # arglocs is call locs + guard_locs, split them - assert fcond == op.numargs() + 3 - call_args = arglocs[:fcond] - guard_locs = arglocs[fcond:] - self._store_force_index(guard_op) - self._emit_call(op, call_args) + if rop.is_call_assembler(op.getopnum()): + if fcond == 4: + [argloc, vloc, result_loc, tmploc] = arglocs[:4] + else: + [argloc, result_loc, tmploc] = arglocs[:3] + vloc = locations.imm(0) + guard_locs = arglocs[fcond:] + self._store_force_index(guard_op) + self.call_assembler(op, argloc, vloc, result_loc, tmploc) + else: + assert fcond == op.numargs() + 3 + call_args = arglocs[:fcond] + guard_locs = arglocs[fcond:] + self._store_force_index(guard_op) + self._emit_call(op, call_args) + # process the guard_not_forced ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.LDR_ri(r.ip0.value, r.fp.value, ofs) self.mc.CMP_ri(r.ip0.value, 0) self._emit_guard(guard_op, c.EQ, guard_locs) + def _call_assembler_emit_call(self, addr, argloc, resloc): + ofs = self.saved_threadlocal_addr + # we are moving the threadlocal directly to x1, to avoid strange + # dances + self.mc.LDR_ri(r.x1.value, r.sp.value, ofs) + self.simple_call(addr, [argloc], result_loc=resloc) + + def _call_assembler_emit_helper_call(self, addr, arglocs, resloc): + self.simple_call(addr, arglocs, result_loc=resloc) + + def _call_assembler_check_descr(self, value, tmploc): + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + self.mc.LDR_ri(r.ip0.value, tmploc.value, ofs) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip0.value, value) + else: + self.mc.gen_load_int(r.ip1.value, value) + self.mc.CMP_rr(r.ip0.value, r.ip1.value) + pos = self.mc.currpos() + self.mc.BRK() + return pos + + def _call_assembler_patch_je(self, result_loc, jmp_location): + pos = self.mc.currpos() + self.mc.BRK() + # + pmc = OverwritingBuilder(self.mc, jmp_location, WORD) + pmc.B_ofs_cond(self.mc.currpos() - jmp_location, c.EQ) + return pos + + def _call_assembler_load_result(self, op, result_loc): + if op.type != 'v': + # load the return value from (tmploc, 0) + kind = op.type + descr = self.cpu.getarraydescr_for_frame(kind) + if kind == FLOAT: + ofs = self.cpu.unpack_arraydescr(descr) + assert check_imm_arg(ofs) + assert result_loc.is_vfp_reg() + # we always have a register here, since we have to sync them + # before call_assembler + self.mc.LDR_di(result_loc.value, r.x0.value, ofs) + else: + assert result_loc is r.x0 + ofs = self.cpu.unpack_arraydescr(descr) + assert check_imm_arg(ofs) + self.mc.LDR_ri(result_loc.value, r.x0.value, ofs) + + def _call_assembler_patch_jmp(self, jmp_location): + # merge point + currpos = self.mc.currpos() + pmc = OverwritingBuilder(self.mc, jmp_location, WORD) + pmc.B_ofs(currpos - jmp_location) + def _store_force_index(self, guard_op): faildescr = guard_op.getdescr() faildescrindex = self.get_gcref_from_faildescr(faildescr) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -640,7 +640,13 @@ if rop.is_call_release_gil(prev_op.getopnum()): arglocs = self._prepare_call(prev_op, save_all_regs=True, first_arg_index=2) + elif rop.is_call_assembler(prev_op.getopnum()): + locs = self.locs_for_call_assembler(prev_op) + tmploc = self.get_scratch_reg(INT, selected_reg=r.x0) + resloc = self._call(prev_op, locs + [tmploc], gc_level=2) + arglocs = locs + [resloc, tmploc] else: + assert rop.is_call_may_force(prev_op.getopnum()) arglocs = self._prepare_call(prev_op, save_all_regs=True) guard_locs = self._guard_impl(op) return arglocs + guard_locs, len(arglocs) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -3722,7 +3722,8 @@ def test_assembler_call(self): called = [] def assembler_helper(deadframe, virtualizable): - assert self.cpu.get_int_value(deadframe, 0) == 97 + print "CALLED ASSEMBLER HELPER" + called.append(self.cpu.get_int_value(deadframe, 0)) called.append(self.cpu.get_latest_descr(deadframe)) return 4 + 9 @@ -3776,7 +3777,7 @@ args = [i+1 for i in range(10)] deadframe = self.cpu.execute_token(othertoken, *args) assert self.cpu.get_int_value(deadframe, 0) == 13 - assert called == [finish_descr] + assert called == [97, finish_descr] # test the fast path, which should not call assembler_helper() del called[:] From pypy.commits at gmail.com Mon Jun 24 04:06:56 2019 From: pypy.commits at gmail.com (fijal) Date: Mon, 24 Jun 2019 01:06:56 -0700 (PDT) Subject: [pypy-commit] pypy arm64: redirect_call_assembler Message-ID: <5d1084a0.1c69fb81.9fa91.dbd3@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96847:4e707c8ce00d Date: 2019-06-22 18:46 +0000 http://bitbucket.org/pypy/pypy/changeset/4e707c8ce00d/ Log: redirect_call_assembler diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -1034,6 +1034,28 @@ self._check_frame_depth(self.mc, self._regalloc.get_gcmap(), expected_size=expected_size) + # ../x86/assembler.py:668 + def redirect_call_assembler(self, oldlooptoken, newlooptoken): + # some minimal sanity checking + old_nbargs = oldlooptoken.compiled_loop_token._debug_nbargs + new_nbargs = newlooptoken.compiled_loop_token._debug_nbargs + assert old_nbargs == new_nbargs + # we overwrite the instructions at the old _ll_function_addr + # to start with a JMP to the new _ll_function_addr. + # Ideally we should rather patch all existing CALLs, but well. + oldadr = oldlooptoken._ll_function_addr + target = newlooptoken._ll_function_addr + # copy frame-info data + baseofs = self.cpu.get_baseofs_of_frame_field() + newlooptoken.compiled_loop_token.update_frame_info( + oldlooptoken.compiled_loop_token, baseofs) + mc = InstrBuilder() + mc.B(target) + mc.copy_to_raw_memory(oldadr) + # + jl.redirect_assembler(oldlooptoken, newlooptoken, newlooptoken.number) + + def not_implemented(msg): msg = '[ARM64/asm] %s\n' % msg diff --git a/rpython/jit/backend/aarch64/runner.py b/rpython/jit/backend/aarch64/runner.py --- a/rpython/jit/backend/aarch64/runner.py +++ b/rpython/jit/backend/aarch64/runner.py @@ -37,6 +37,9 @@ operations, original_loop_token, log=log) + def redirect_call_assembler(self, oldlooptoken, newlooptoken): + self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) + def cast_ptr_to_int(x): adr = llmemory.cast_ptr_to_adr(x) return CPU_ARM64.cast_adr_to_int(adr) From pypy.commits at gmail.com Mon Jun 24 04:06:57 2019 From: pypy.commits at gmail.com (fijal) Date: Mon, 24 Jun 2019 01:06:57 -0700 (PDT) Subject: [pypy-commit] pypy arm64: some more obscure loads Message-ID: <5d1084a1.1c69fb81.7ef1b.eef5@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96848:5b330a4bfb72 Date: 2019-06-22 18:58 +0000 http://bitbucket.org/pypy/pypy/changeset/5b330a4bfb72/ Log: some more obscure loads diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -116,10 +116,16 @@ def LDR_ri(self, rt, rn, immed): base = 0b1111100101 - assert 0 <= immed <= 1<<15 + assert 0 <= immed < 1<<16 assert immed & 0x7 == 0 self.write32((base << 22) | (immed >> 3 << 10) | (rn << 5) | rt) + def LDR_uint32_ri(self, rt, rn, immed): + base = 0b1011100101 + assert 0 <= immed < 1<<15 + assert immed & 0x3 == 0 + self.write32((base << 22) | (immed >> 2 << 10) | (rn << 5) | rt) + def LDR_di(self, rt, rn, offset): assert offset & 0x7 == 0 assert 0 <= offset < 32768 @@ -132,7 +138,7 @@ def LDRB_ri(self, rt, rn, immed): base = 0b0011100101 - assert 0 <= immed <= 1<<12 + assert 0 <= immed < 1<<12 self.write32((base << 22) | (immed << 10) | (rn << 5) | rt) def LDRSH_ri(self, rt, rn, immed): @@ -153,6 +159,12 @@ base = 0b01111000011 self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt) + def LDRH_ri(self, rt, rn, immed): + assert immed & 0b1 == 0 + assert 0 <= immed < (1 << 13) + base = 0b0111100101 + self.write32((base << 22) | (immed >> 1 << 10) | (rn << 5) | rt) + def LDRB_rr(self, rt, rn, rm): base = 0b00111000011 self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt) @@ -161,6 +173,12 @@ base = 0b10111000101 self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt) + def LDRSW_ri(self, rt, rn, immed): + assert immed & 0x3 == 0 + assert 0 <= immed < (1<<14) + base = 0b1011100110 + self.write32((base << 22) | (immed >> 2 << 10) | (rn << 5) | rt) + def LDRSH_rr(self, rt, rn, rm): base = 0b01111000101 self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt) @@ -169,6 +187,11 @@ base = 0b00111000101 self.write32((base << 21) | (rm << 16) | (0b011010 << 10) | (rn << 5) | rt) + def LDRSB_ri(self, rt, rn, immed): + base = 0b0011100110 + assert 0 <= immed < 1 << 12 + self.write32((base << 22) | (immed << 10) | (rn << 5) | rt) + def LDR_r_literal(self, rt, offset): base = 0b01011000 assert -(1 << 20) <= offset < (1<< 20) From pypy.commits at gmail.com Mon Jun 24 04:06:59 2019 From: pypy.commits at gmail.com (fijal) Date: Mon, 24 Jun 2019 01:06:59 -0700 (PDT) Subject: [pypy-commit] pypy arm64: converting bytes of longlong and float Message-ID: <5d1084a3.1c69fb81.ad9a7.6011@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96849:67646e01619f Date: 2019-06-24 08:06 +0000 http://bitbucket.org/pypy/pypy/changeset/67646e01619f/ Log: converting bytes of longlong and float diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -63,6 +63,14 @@ def MOV_rr(self, rd, rn): self.ORR_rr(rd, r.xzr.value, rn) + def UMOV_rd(self, rd, rn): + base = 0b0100111000001000001111 + self.write32((base << 10) | (rn << 5) | rd) + + def INS_dr(self, rd, rn): + base = 0b0100111000001000000111 + self.write32((base << 10) | (rn << 5) | rd) + def ORR_rr(self, rd, rn, rm): base = 0b10101010000 self.write32((base << 21) | (rm << 16) | diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -263,7 +263,15 @@ def emit_op_float_truediv(self, op, arglocs): arg1, arg2, res = arglocs - self.mc.FDIV_dd(res.value, arg1.value, arg2.value) + self.mc.FDIV_dd(res.value, arg1.value, arg2.value) + + def emit_op_convert_float_bytes_to_longlong(self, op, arglocs): + arg, res = arglocs + self.mc.UMOV_rd(res.value, arg.value) + + def emit_op_convert_longlong_bytes_to_float(self, op, arglocs): + arg, res = arglocs + self.mc.INS_dr(res.value, arg.value) emit_op_float_lt = gen_float_comp_op('float_lt', c.VFP_LT) emit_op_float_le = gen_float_comp_op('float_le', c.VFP_LE) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -485,6 +485,8 @@ prepare_op_cast_float_to_int = prepare_unary prepare_op_cast_int_to_float = prepare_unary + prepare_op_convert_float_bytes_to_longlong = prepare_unary + prepare_op_convert_longlong_bytes_to_float = prepare_unary # --------------------------------- fields -------------------------- From pypy.commits at gmail.com Mon Jun 24 05:18:32 2019 From: pypy.commits at gmail.com (mattip) Date: Mon, 24 Jun 2019 02:18:32 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: test, refactor to more closely match cpython Message-ID: <5d109568.1c69fb81.a0a10.d9b2@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96850:da92ca675f66 Date: 2019-06-24 12:16 +0300 http://bitbucket.org/pypy/pypy/changeset/da92ca675f66/ Log: test, refactor to more closely match cpython diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -165,18 +165,19 @@ view = self.view length = space.len_w(w_index) ndim = view.getndim() - if length < ndim: + if is_multislice(space, w_index): raise oefmt(space.w_NotImplementedError, + "multi-dimensional slicing is not implemented") + elif length != ndim: + if length < ndim: + raise oefmt(space.w_NotImplementedError, "sub-views are not implemented") - if length > ndim: - raise oefmt(space.w_TypeError, \ + elif length > ndim: + raise oefmt(space.w_TypeError, \ "cannot index %d-dimension view with %d-element tuple", length, ndim) - elif is_multislice(space, w_index): - raise oefmt(space.w_NotImplementedError, - "multi-dimensional slicing is not implemented") start = self._start_from_tuple(space, w_index) itemsize = self.getitemsize() val = self.view.bytes_from_value(space, w_obj) diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -176,6 +176,7 @@ assert m[0] == 0 m[0] = 1 assert m[0] == 1 + raises(NotImplementedError, m.__setitem__, (slice(0,1,1), slice(0,1,2)), 0) def test_int_array_slice(self): import array From pypy.commits at gmail.com Tue Jun 25 06:27:50 2019 From: pypy.commits at gmail.com (mattip) Date: Tue, 25 Jun 2019 03:27:50 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: test, fix for non-byte n-d setitem Message-ID: <5d11f726.1c69fb81.77e40.cc5e@mx.google.com> Author: Matti Picus Branch: py3.6 Changeset: r96851:6c4d7cc5d649 Date: 2019-06-25 13:27 +0300 http://bitbucket.org/pypy/pypy/changeset/6c4d7cc5d649/ Log: test, fix for non-byte n-d setitem diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py --- a/pypy/objspace/std/memoryobject.py +++ b/pypy/objspace/std/memoryobject.py @@ -179,9 +179,8 @@ length, ndim) start = self._start_from_tuple(space, w_index) - itemsize = self.getitemsize() val = self.view.bytes_from_value(space, w_obj) - self.view.setbytes(start * itemsize, val) + self.view.setbytes(start, val) def _decode_index(self, space, w_index, is_slice): shape = self.getshape() diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py --- a/pypy/objspace/std/test/test_memoryobject.py +++ b/pypy/objspace/std/test/test_memoryobject.py @@ -289,20 +289,19 @@ assert m2.shape == m1.shape def test_2d(self): - m = memoryview(bytearray(b'1234123412341234')) - assert m[3] == ord('4') - m[3] = ord('z') - assert m[3] == ord('z') - m = m.cast('B', shape=(4, 4)) - assert m[2, 3] == ord('4') - m[2, 3] = ord('a') - assert m[2, 3] == ord('a') - raises(TypeError, m.__setitem__, (2, 3), bytearray(b'12')) + import struct + a = list(range(16)) + ba = bytearray(struct.pack("%di" % len(a), *a)) + m = memoryview(ba).cast("i", shape=(4, 4)) + assert m[2, 3] == 11 + m[2, 3] = -1 + assert m[2, 3] == -1 + raises(TypeError, m.__setitem__, (2, 3), 'a') # slices in 2d memoryviews are not supported at all raises(TypeError, m.__getitem__, (slice(None), 3)) - raises(TypeError, m.__setitem__, (slice(None), 3), ord('a')) + raises(TypeError, m.__setitem__, (slice(None), 3), 123) raises(NotImplementedError, m.__getitem__, (slice(0,1,1), slice(0,1,2))) - raises(NotImplementedError, m.__setitem__, (slice(0,1,1), slice(0,1,2)), ord('a')) + raises(NotImplementedError, m.__setitem__, (slice(0,1,1), slice(0,1,2)), 123) class AppTestCtypes(object): spaceconfig = dict(usemodules=['sys', '_rawffi']) From pypy.commits at gmail.com Tue Jun 25 07:13:30 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 04:13:30 -0700 (PDT) Subject: [pypy-commit] pypy arm64: guard_not_invalidated Message-ID: <5d1201da.1c69fb81.d0444.3d5f@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96852:caa3d6c5a7fe Date: 2019-06-24 08:36 +0000 http://bitbucket.org/pypy/pypy/changeset/caa3d6c5a7fe/ Log: guard_not_invalidated diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -710,7 +710,6 @@ mc.B_ofs_cond(relative_offset, c.get_opposite_of(tok.fcond)) mc.copy_to_raw_memory(guard_pos) else: - XX clt.invalidate_positions.append((guard_pos, relative_offset)) def fixup_target_tokens(self, rawstart): @@ -895,8 +894,13 @@ XXX def _mov_imm_to_loc(self, prev_loc, loc): - assert loc.is_core_reg() - self.mc.gen_load_int(loc.value, prev_loc.value) + if loc.is_core_reg(): + self.mc.gen_load_int(loc.value, prev_loc.value) + elif loc.is_stack(): + self.mc.gen_load_int(r.ip0.value, prev_loc.value) + self.mc.STR_ri(r.ip0.value, r.fp.value, loc.value) + else: + assert False def new_stack_loc(self, i, tp): base_ofs = self.cpu.get_baseofs_of_frame_field() diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -502,6 +502,8 @@ self._emit_guard(guard_op, c.get_opposite_of(fcond), arglocs) emit_guard_op_guard_overflow = emit_guard_op_guard_false + def emit_op_guard_not_invalidated(self, op, arglocs): + self._emit_guard(op, 0, arglocs, True) def load_condition_into_cc(self, loc): if not loc.is_core_reg(): diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -817,6 +817,9 @@ prepare_op_guard_gc_type = prepare_op_guard_class prepare_op_guard_subclass = prepare_op_guard_class + def prepare_op_guard_not_invalidated(self, op): + return self._guard_impl(op) + def prepare_op_guard_exception(self, op): boxes = op.getarglist() arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) diff --git a/rpython/jit/backend/aarch64/runner.py b/rpython/jit/backend/aarch64/runner.py --- a/rpython/jit/backend/aarch64/runner.py +++ b/rpython/jit/backend/aarch64/runner.py @@ -4,6 +4,7 @@ from rpython.jit.backend.aarch64 import registers as r from rpython.jit.backend.aarch64.regalloc import VFPRegisterManager from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU +from rpython.jit.backend.aarch64.codebuilder import InstrBuilder class CPU_ARM64(AbstractLLCPU): """ARM 64""" @@ -40,6 +41,21 @@ def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) + def invalidate_loop(self, looptoken): + """Activate all GUARD_NOT_INVALIDATED in the loop and its attached + bridges. Before this call, all GUARD_NOT_INVALIDATED do nothing; + after this call, they all fail. Note that afterwards, if one such + guard fails often enough, it has a bridge attached to it; it is + possible then to re-call invalidate_loop() on the same looptoken, + which must invalidate all newer GUARD_NOT_INVALIDATED, but not the + old one that already has a bridge attached to it.""" + for jmp, tgt in looptoken.compiled_loop_token.invalidate_positions: + mc = InstrBuilder() + mc.B_ofs(tgt) + mc.copy_to_raw_memory(jmp) + # positions invalidated + looptoken.compiled_loop_token.invalidate_positions = [] + def cast_ptr_to_int(x): adr = llmemory.cast_ptr_to_adr(x) return CPU_ARM64.cast_adr_to_int(adr) From pypy.commits at gmail.com Tue Jun 25 07:13:32 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 04:13:32 -0700 (PDT) Subject: [pypy-commit] pypy arm64: math sqrt Message-ID: <5d1201dc.1c69fb81.41d7d.705d@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96853:03ea4ff1aa71 Date: 2019-06-24 08:54 +0000 http://bitbucket.org/pypy/pypy/changeset/03ea4ff1aa71/ Log: math sqrt diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -246,6 +246,10 @@ base = 0b1001111001100010 self.write32((base << 16) | (rn << 5) | rd) + def FSQRT_dd(self, rd, rn): + base = 0b0001111001100001110000 + self.write32((base << 10) | (rn << 5) | rd) + def SUB_rr(self, rd, rn, rm, s=0): base = 0b11001011001 | (s << 8) self.write32((base << 21) | (rm << 16) | (0b11 << 13) | (rn << 5) | (rd)) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -273,6 +273,10 @@ arg, res = arglocs self.mc.INS_dr(res.value, arg.value) + def math_sqrt(self, op, arglocs): + arg, res = arglocs + self.mc.FSQRT_dd(res.value, arg.value) + emit_op_float_lt = gen_float_comp_op('float_lt', c.VFP_LT) emit_op_float_le = gen_float_comp_op('float_le', c.VFP_LE) emit_op_float_eq = gen_float_comp_op('float_eq', c.EQ) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -485,6 +485,12 @@ prepare_op_cast_float_to_int = prepare_unary prepare_op_cast_int_to_float = prepare_unary + def _prepare_op_math_sqrt(self, op): + loc1 = self.make_sure_var_in_reg(op.getarg(1)) + self.possibly_free_vars_for_op(op) + res = self.force_allocate_reg(op) + return [loc1, res] + prepare_op_convert_float_bytes_to_longlong = prepare_unary prepare_op_convert_longlong_bytes_to_float = prepare_unary @@ -563,6 +569,7 @@ EffectInfo.OS_LLONG_AND, EffectInfo.OS_LLONG_OR, EffectInfo.OS_LLONG_XOR): + XXX if self.cpu.cpuinfo.neon: args = self._prepare_llong_binop_xx(op, fcond) self.perform_extra(op, args, fcond) @@ -572,8 +579,8 @@ self.perform_extra(op, args, fcond) return elif oopspecindex == EffectInfo.OS_MATH_SQRT: - args = self._prepare_op_math_sqrt(op, fcond) - self.perform_extra(op, args, fcond) + args = self._prepare_op_math_sqrt(op) + self.assembler.math_sqrt(op, args) return elif oopspecindex == EffectInfo.OS_THREADLOCALREF_GET: args = self._prepare_threadlocalref_get(op, fcond) From pypy.commits at gmail.com Tue Jun 25 07:13:33 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 04:13:33 -0700 (PDT) Subject: [pypy-commit] pypy arm64: a few more obscure operations Message-ID: <5d1201dd.1c69fb81.535b8.7eac@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96854:d11024632dda Date: 2019-06-24 10:08 +0000 http://bitbucket.org/pypy/pypy/changeset/d11024632dda/ Log: a few more obscure operations diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -246,6 +246,18 @@ base = 0b1001111001100010 self.write32((base << 16) | (rn << 5) | rd) + def SXTB_rr(self, rd, rn): + base = 0b1001001101000000000111 + self.write32((base << 10) | (rn << 5) | rd) + + def SXTH_rr(self, rd, rn): + base = 0b1001001101000000001111 + self.write32((base << 10) | (rn << 5) | rd) + + def SXTW_rr(self, rd, rn): + base = 0b1001001101000000011111 + self.write32((base << 10) | (rn << 5) | rd) + def FSQRT_dd(self, rd, rn): base = 0b0001111001100001110000 self.write32((base << 10) | (rn << 5) | rd) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -228,6 +228,26 @@ reg, res = arglocs self.mc.MVN_rr(res.value, reg.value) + def emit_op_int_force_ge_zero(self, op, arglocs): + arg, res = arglocs + self.mc.MOVZ_r_u16(res.value, 0, 0) + self.mc.CMP_ri(arg.value, 0) + self.mc.B_ofs_cond(8, c.LT) # jump over the next instruction + self.mc.MOV_rr(res.value, arg.value) + # jump here + + def emit_op_int_signext(self, op, arglocs): + arg, numbytes, res = arglocs + assert numbytes.is_imm() + if numbytes.value == 1: + self.mc.SXTB_rr(res.value, arg.value) + elif numbytes.value == 2: + self.mc.SXTH_rr(res.value, arg.value) + elif numbytes.value == 4: + self.mc.SXTW_rr(res.value, arg.value) + else: + raise AssertionError("bad number of bytes") + def emit_op_increment_debug_counter(self, op, arglocs): return # XXXX base_loc, value_loc = arglocs diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -371,6 +371,17 @@ def prepare_comp_op_int_mul_ovf(self, op, res_in_cc): return self.prepare_op_int_mul(op) + def prepare_op_int_force_ge_zero(self, op): + argloc = self.make_sure_var_in_reg(op.getarg(0)) + resloc = self.force_allocate_reg(op, [op.getarg(0)]) + return [argloc, resloc] + + def prepare_op_int_signext(self, op): + argloc = self.make_sure_var_in_reg(op.getarg(0)) + numbytes = op.getarg(1).getint() + resloc = self.force_allocate_reg(op) + return [argloc, imm(numbytes), resloc] + # some of those have forms of imm that they accept, but they're rather # obscure. Can be future optimization prepare_op_int_and = prepare_op_int_mul From pypy.commits at gmail.com Tue Jun 25 07:13:35 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 04:13:35 -0700 (PDT) Subject: [pypy-commit] pypy arm64: slow progress Message-ID: <5d1201df.1c69fb81.a90a1.da9b@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96855:8003825bd555 Date: 2019-06-24 11:36 +0000 http://bitbucket.org/pypy/pypy/changeset/8003825bd555/ Log: slow progress diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -404,8 +404,7 @@ # pop the values passed on the stack, gcmap -> r0, expected_size -> r1 mc.LDP_rri(r.x0.value, r.x1.value, r.sp.value, 0) - # XXX # store return address and keep the stack aligned - # mc.PUSH([r.ip.value, r.lr.value]) + mc.STR_ri(r.lr.value, r.sp.value, 0) # store the current gcmap(r0) in the jitframe gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') @@ -416,7 +415,7 @@ # store a possibly present exception # we use a callee saved reg here as a tmp for the exc. - self._store_and_reset_exception(mc, None, r.x19, on_frame=True) + self._store_and_reset_exception(mc, None, r.ip1, on_frame=True) # call realloc_frame, it takes two arguments # arg0: the old jitframe @@ -428,10 +427,11 @@ mc.MOV_rr(r.fp.value, r.x0.value) # restore a possibly present exception - self._restore_exception(mc, None, r.x19) + self._restore_exception(mc, None, r.ip1) gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: + xxx self._load_shadowstack_top(mc, r.r5, gcrootmap) # store the new jitframe addr in the shadowstack mc.STR_ri(r.x0.value, r.r5.value, imm=-WORD) @@ -444,8 +444,8 @@ self._pop_all_regs_from_jitframe(mc, [], self.cpu.supports_floats) # return + mc.LDR_ri(r.lr.value, r.sp.value, 0) mc.ADD_ri(r.sp.value, r.sp.value, 2*WORD) - mc.LDR_ri(r.lr.value, r.sp.value, WORD) mc.RET_r(r.lr.value) self._frame_realloc_slowpath = mc.materialize(self.cpu, []) @@ -572,15 +572,15 @@ for _ in range(mc.get_max_size_of_gen_load_int()): mc.NOP() else: - mc.gen_load_int(r.lr.value, expected_size) - mc.CMP_rr(r.ip0.value, r.lr.value) + mc.gen_load_int(r.ip1.value, expected_size) + mc.CMP_rr(r.ip0.value, r.ip1.value) jg_location = mc.currpos() mc.BRK() - # the size value is still stored in lr + # the size value is still stored in ip1 mc.SUB_ri(r.sp.value, r.sp.value, 2*WORD) - mc.STR_ri(r.lr.value, r.sp.value, WORD) + mc.STR_ri(r.ip1.value, r.sp.value, WORD) mc.gen_load_int(r.ip0.value, rffi.cast(lltype.Signed, gcmap)) mc.STR_ri(r.ip0.value, r.sp.value, 0) @@ -665,7 +665,7 @@ def patch_stack_checks(self, framedepth, rawstart): for ofs in self.frame_depth_to_patch: mc = InstrBuilder() - mc.gen_load_int(r.lr.value, framedepth) + mc.gen_load_int(r.ip1.value, framedepth) mc.copy_to_raw_memory(ofs + rawstart) def load_from_gc_table(self, regnum, index): @@ -914,6 +914,9 @@ self.mc.STR_ri(r.ip0.value, r.sp.value, pos) elif loc.is_vfp_reg(): xxx + elif loc.is_imm(): + self.mc.gen_load_int(r.ip0.value, loc.value) + self.mc.STR_ri(r.ip0.value, r.sp.value, pos) else: assert False, "wrong loc" diff --git a/rpython/jit/backend/aarch64/test/test_runner.py b/rpython/jit/backend/aarch64/test/test_runner.py --- a/rpython/jit/backend/aarch64/test/test_runner.py +++ b/rpython/jit/backend/aarch64/test/test_runner.py @@ -25,20 +25,11 @@ # for the individual tests see # ====> ../../test/runner_test.py - #add_loop_instructions = 'ldr; adds; cmp; beq; b;' - #if arch_version == 7: - # bridge_loop_instructions = ('ldr; movw; nop; cmp; bge; ' - # 'push; movw; movt; push; movw; movt; ' - # 'blx; movw; movt; bx;') - #else: - # bridge_loop_instructions = ('ldr; mov; nop; nop; nop; cmp; bge; ' - # 'push; ldr; mov; ' - # '[^;]+; ' # inline constant - # 'push; ldr; mov; ' - # '[^;]+; ' # inline constant - # 'blx; ldr; mov; ' - # '[^;]+; ' # inline constant - # 'bx;') + add_loop_instructions = 'ldr; add; cmp; b.eq; b; brk;' + bridge_loop_instructions = ('ldr; mov; nop; nop; nop; ' + 'cmp; b.ge; sub; str; mov; movk; ' + 'str; mov; movk; movk; movk; blr; ' + 'mov; movk; movk; movk; br; brk;') def get_cpu(self): cpu = CPU(rtyper=None, stats=FakeStats()) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -4972,7 +4972,7 @@ EffectInfo.MOST_GENERAL) def func2(a, b, c, d, e, f, g, h, i, j, k, l): - pass + print "CALLED" FUNC2 = self.FuncType([lltype.Signed] * 12, lltype.Void) FPTR2 = self.Ptr(FUNC2) diff --git a/rpython/jit/backend/tool/viewcode.py b/rpython/jit/backend/tool/viewcode.py --- a/rpython/jit/backend/tool/viewcode.py +++ b/rpython/jit/backend/tool/viewcode.py @@ -48,6 +48,7 @@ 'i386': 'i386', 'arm': 'arm', 'arm_32': 'arm', + 'aarch64': 'aarch64', 'ppc' : 'powerpc:common64', 'ppc-64' : 'powerpc:common64', 's390x': 's390:64-bit', From pypy.commits at gmail.com Tue Jun 25 07:13:37 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 04:13:37 -0700 (PDT) Subject: [pypy-commit] pypy arm64: jump with float constants Message-ID: <5d1201e1.1c69fb81.8d488.5d8e@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96856:58ffd4498657 Date: 2019-06-24 11:40 +0000 http://bitbucket.org/pypy/pypy/changeset/58ffd4498657/ Log: jump with float constants diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -930,8 +930,13 @@ assert False, "wrong loc" def _mov_imm_float_to_loc(self, prev_loc, loc): - assert loc.is_vfp_reg() - self.load(loc, prev_loc) + if loc.is_vfp_reg(): + self.load(loc, prev_loc) + elif loc.is_stack(): + self.load(r.vfp_ip, prev_loc) + self._mov_vfp_reg_to_loc(r.vfp_ip, loc) + else: + assert False, "wrong loc" def _mov_vfp_reg_to_loc(self, prev_loc, loc): assert loc.is_stack() From pypy.commits at gmail.com Tue Jun 25 07:13:38 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 04:13:38 -0700 (PDT) Subject: [pypy-commit] pypy arm64: zero_array, increment debug counter and exception handling Message-ID: <5d1201e2.1c69fb81.134a6.506b@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96857:c89b27574833 Date: 2019-06-25 11:12 +0000 http://bitbucket.org/pypy/pypy/changeset/c89b27574833/ Log: zero_array, increment debug counter and exception handling diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -405,6 +405,7 @@ mc.LDP_rri(r.x0.value, r.x1.value, r.sp.value, 0) mc.STR_ri(r.lr.value, r.sp.value, 0) + mc.STR_ri(r.x19.value, r.sp.value, WORD) # store the current gcmap(r0) in the jitframe gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') @@ -414,8 +415,7 @@ mc.MOV_rr(r.x0.value, r.fp.value) # store a possibly present exception - # we use a callee saved reg here as a tmp for the exc. - self._store_and_reset_exception(mc, None, r.ip1, on_frame=True) + self._store_and_reset_exception(mc, None, r.x19, on_frame=True) # call realloc_frame, it takes two arguments # arg0: the old jitframe @@ -427,7 +427,7 @@ mc.MOV_rr(r.fp.value, r.x0.value) # restore a possibly present exception - self._restore_exception(mc, None, r.ip1) + self._restore_exception(mc, None, r.x19) gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: @@ -445,6 +445,7 @@ # return mc.LDR_ri(r.lr.value, r.sp.value, 0) + mc.LDR_ri(r.x19.value, r.sp.value, WORD) mc.ADD_ri(r.sp.value, r.sp.value, 2*WORD) mc.RET_r(r.lr.value) self._frame_realloc_slowpath = mc.materialize(self.cpu, []) diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -60,6 +60,15 @@ base = 0b11100100 self.write32((scale << 30) | (base << 22) | (imm >> scale << 10) | (rn << 5) | rt) + def STRB_ri(self, rt, rn, imm): + self.STR_size_ri(0, rt, rn, imm) + + def STRH_ri(self, rt, rn, imm): + self.STR_size_ri(1, rt, rn, imm) + + def STRW_ri(self, rt, rn, imm): + self.STR_size_ri(2, rt, rn, imm) + def MOV_rr(self, rd, rn): self.ORR_rr(rd, r.xzr.value, rn) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -249,7 +249,6 @@ raise AssertionError("bad number of bytes") def emit_op_increment_debug_counter(self, op, arglocs): - return # XXXX base_loc, value_loc = arglocs self.mc.LDR_ri(value_loc.value, base_loc.value, 0) self.mc.ADD_ri(value_loc.value, value_loc.value, 1) @@ -635,6 +634,111 @@ self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs, array=True) + #from ../x86/regalloc.py:1388 + def emit_op_zero_array(self, op, arglocs): + from rpython.jit.backend.llsupport.descr import unpack_arraydescr + assert len(arglocs) == 0 + size_box = op.getarg(2) + if isinstance(size_box, ConstInt) and size_box.getint() == 0: + return + itemsize, baseofs, _ = unpack_arraydescr(op.getdescr()) + args = op.getarglist() + # + # ZERO_ARRAY(base_loc, start, size, 1, 1) + # 'start' and 'size' are both expressed in bytes, + # and the two scaling arguments should always be ConstInt(1) on ARM. + assert args[3].getint() == 1 + assert args[4].getint() == 1 + # + base_loc = self._regalloc.rm.make_sure_var_in_reg(args[0], args) + startbyte_box = args[1] + if isinstance(startbyte_box, ConstInt): + startbyte_loc = None + startbyte = startbyte_box.getint() + assert startbyte >= 0 + else: + startbyte_loc = self._regalloc.rm.make_sure_var_in_reg(startbyte_box, + args) + startbyte = -1 + + # base_loc and startbyte_loc are in two regs here (or startbyte_loc + # is an immediate). Compute the dstaddr_loc, which is the raw + # address that we will pass as first argument to memset(). + # It can be in the same register as either one, but not in + # args[2], because we're still needing the latter. + dstaddr_loc = r.ip1 + if startbyte >= 0: # a constant + ofs = baseofs + startbyte + reg = base_loc.value + else: + self.mc.ADD_rr(dstaddr_loc.value, + base_loc.value, startbyte_loc.value) + ofs = baseofs + reg = dstaddr_loc.value + if check_imm_arg(ofs): + self.mc.ADD_ri(dstaddr_loc.value, reg, ofs) + else: + self.mc.gen_load_int(r.ip0.value, ofs) + self.mc.ADD_rr(dstaddr_loc.value, reg, r.ip0.value) + + # We use STRB, STRH, STRW or STR based on whether we know the array + # item size is a multiple of 1, 2 or 4. + if itemsize & 1: itemsize = 1 + elif itemsize & 2: itemsize = 2 + elif itemsize & 4: itemsize = 4 + else: itemsize = 8 + limit = itemsize + next_group = -1 + if itemsize < 8 and startbyte >= 0: + # we optimize STRB/STRH into STR, but this needs care: + # it only works if startindex_loc is a constant, otherwise + # we'd be doing unaligned accesses. + next_group = (-startbyte) & 7 + limit = 8 + + if (isinstance(size_box, ConstInt) and + size_box.getint() <= 14 * limit): # same limit as GCC + # Inline a series of STR operations, starting at 'dstaddr_loc'. + # + self.mc.gen_load_int(r.ip0.value, 0) + i = 0 + adjustment = 0 + needs_adjustment = itemsize < 8 and (startbyte % 8) + total_size = size_box.getint() + while i < total_size: + sz = itemsize + if i == next_group: + next_group += 8 + if next_group <= total_size: + sz = 8 + if sz == 8: + if needs_adjustment: + self.mc.ADD_ri(dstaddr_loc.value, dstaddr_loc.value, i) + adjustment = -i + needs_adjustment = False + self.mc.STR_ri(r.ip0.value, dstaddr_loc.value, i + adjustment) + elif sz == 4: + self.mc.STRW_ri(r.ip0.value, dstaddr_loc.value, i + adjustment) + elif sz == 2: + self.mc.STRH_ri(r.ip0.value, dstaddr_loc.value, i + adjustment) + else: + self.mc.STRB_ri(r.ip0.value, dstaddr_loc.value, i + adjustment) + i += sz + + else: + if isinstance(size_box, ConstInt): + size_loc = self.imm(size_box.getint()) + else: + # load size_loc in a register different than dstaddr_loc + size_loc = self._regalloc.rm.make_sure_var_in_reg(size_box, + []) + # + # call memset() + self._regalloc.before_call() + self.simple_call_no_collect(self.imm(self.memset_addr), + [dstaddr_loc, self.imm(0), size_loc]) + self._regalloc.rm.possibly_free_var(size_box) + def _emit_op_cond_call(self, op, arglocs, fcond): if len(arglocs) == 2: res_loc = arglocs[1] # cond_call_value @@ -859,6 +963,10 @@ result_size) cb.emit() + def simple_call_no_collect(self, fnloc, arglocs): + cb = Aarch64CallBuilder(self, fnloc, arglocs) + cb.emit_no_collect() + def emit_guard_op_guard_not_forced(self, op, guard_op, fcond, arglocs): # arglocs is call locs + guard_locs, split them if rop.is_call_assembler(op.getopnum()): diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -313,6 +313,7 @@ prepare_op_jit_debug = void prepare_op_enter_portal_frame = void prepare_op_leave_portal_frame = void + prepare_op_zero_array = void # dealth with in opassembler.py def prepare_int_ri(self, op, res_in_cc): boxes = op.getarglist() @@ -648,6 +649,10 @@ resloc = self.after_call(op) return resloc + def before_call(self, save_all_regs=False): + self.rm.before_call(save_all_regs=save_all_regs) + self.vfprm.before_call(save_all_regs=save_all_regs) + def after_call(self, v): if v.type == 'v': return From pypy.commits at gmail.com Tue Jun 25 07:33:35 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 04:33:35 -0700 (PDT) Subject: [pypy-commit] pypy arm64: Small changes from test_random Message-ID: <5d12068f.1c69fb81.4f712.c2f7@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96858:a524391486ad Date: 2019-06-25 11:32 +0000 http://bitbucket.org/pypy/pypy/changeset/a524391486ad/ Log: Small changes from test_random diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -940,8 +940,12 @@ assert False, "wrong loc" def _mov_vfp_reg_to_loc(self, prev_loc, loc): - assert loc.is_stack() - self.mc.STR_di(prev_loc.value, r.fp.value, loc.value) + if loc.is_stack(): + self.mc.STR_di(prev_loc.value, r.fp.value, loc.value) + elif loc.is_vfp_reg(): + self.mc.MOV_dd(prev_loc.value, loc.value) + else: + assert False, "wrong loc" def push_locations(self, locs): if not locs: diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -72,6 +72,11 @@ def MOV_rr(self, rd, rn): self.ORR_rr(rd, r.xzr.value, rn) + def MOV_dd(self, rd, rn): + base = 0b00001110101 + self.write32((base << 21) | (rn << 16) | (0b000111 << 10) | (rn << 5) | + rd) + def UMOV_rd(self, rd, rn): base = 0b0100111000001000001111 self.write32((base << 10) | (rn << 5) | rd) From pypy.commits at gmail.com Tue Jun 25 07:33:37 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 04:33:37 -0700 (PDT) Subject: [pypy-commit] pypy arm64: start working on call_release_gil Message-ID: <5d120691.1c69fb81.6c9bd.a335@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96859:dc101f8cce32 Date: 2019-06-25 11:32 +0000 http://bitbucket.org/pypy/pypy/changeset/dc101f8cce32/ Log: start working on call_release_gil diff --git a/rpython/jit/backend/aarch64/callbuilder.py b/rpython/jit/backend/aarch64/callbuilder.py --- a/rpython/jit/backend/aarch64/callbuilder.py +++ b/rpython/jit/backend/aarch64/callbuilder.py @@ -149,7 +149,55 @@ xxx def move_real_result_and_call_reacqgil_addr(self, fastgil): - xxx + # try to reacquire the lock. + # XXX r5 == &root_stack_top + # r6 == fastgil + # XXX r7 == previous value of root_stack_top + self.mc.gen_load_int(r.ip1.value, fastgil) + self.mc.LDREX(r.x0.value, r.r6.value) # load the lock value + self.mc.MOV_ri(r.ip.value, 1) + self.mc.CMP_ri(r.r3.value, 0) # is the lock free? + self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ) + # try to claim the lock + self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed? + if self.asm.cpu.cpuinfo.arch_version >= 7: + self.mc.DMB() + # the success of the lock acquisition is defined by + # 'EQ is true', or equivalently by 'r3 == 0'. + # + if self.asm.cpu.gc_ll_descr.gcrootmap: + # When doing a call_release_gil with shadowstack, there + # is the risk that the 'rpy_fastgil' was free but the + # current shadowstack can be the one of a different + # thread. So here we check if the shadowstack pointer + # is still the same as before we released the GIL (saved + # in 'r7'), and if not, we fall back to 'reacqgil_addr'. + self.mc.LDR_ri(r.ip.value, r.r5.value, cond=c.EQ) + self.mc.CMP_rr(r.ip.value, r.r7.value, cond=c.EQ) + b1_location = self.mc.currpos() + self.mc.BKPT() # BEQ below + # there are two cases here: either EQ was false from + # the beginning, or EQ was true at first but the CMP + # made it false. In the second case we need to + # release the fastgil here. We know which case it is + # by checking again r3. + self.mc.CMP_ri(r.r3.value, 0) + self.mc.STR_ri(r.r3.value, r.r6.value, cond=c.EQ) + else: + b1_location = self.mc.currpos() + self.mc.BKPT() # BEQ below + # + # save the result we just got + gpr_to_save, vfp_to_save = self.get_result_locs() + with saved_registers(self.mc, gpr_to_save, vfp_to_save): + self.mc.BL(self.asm.reacqgil_addr) + + # replace b1_location with B(here, c.EQ) + pmc = OverwritingBuilder(self.mc, b1_location, WORD) + pmc.B_offs(self.mc.currpos(), c.EQ) + + if not we_are_translated(): # for testing: now we can accesss + self.mc.SUB_ri(r.fp.value, r.fp.value, 1) # fp again def get_result_locs(self): if self.resloc is None: From pypy.commits at gmail.com Tue Jun 25 10:18:20 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 07:18:20 -0700 (PDT) Subject: [pypy-commit] pypy arm64: (arigo, fijal) call_release_gil first steps Message-ID: <5d122d2c.1c69fb81.8001d.fcd3@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96860:62f3abe7cb9a Date: 2019-06-25 14:17 +0000 http://bitbucket.org/pypy/pypy/changeset/62f3abe7cb9a/ Log: (arigo, fijal) call_release_gil first steps diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -914,7 +914,7 @@ self.mc.LDR_ri(r.ip0.value, r.fp.value, loc.value) self.mc.STR_ri(r.ip0.value, r.sp.value, pos) elif loc.is_vfp_reg(): - xxx + self.mc.STR_di(loc.value, r.sp.value, pos) elif loc.is_imm(): self.mc.gen_load_int(r.ip0.value, loc.value) self.mc.STR_ri(r.ip0.value, r.sp.value, pos) diff --git a/rpython/jit/backend/aarch64/callbuilder.py b/rpython/jit/backend/aarch64/callbuilder.py --- a/rpython/jit/backend/aarch64/callbuilder.py +++ b/rpython/jit/backend/aarch64/callbuilder.py @@ -3,6 +3,7 @@ from rpython.jit.backend.aarch64.arch import WORD from rpython.jit.metainterp.history import INT, FLOAT, REF from rpython.jit.backend.aarch64 import registers as r +from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.aarch64.jump import remap_frame_layout # we use arm algo from rpython.rlib.objectmodel import we_are_translated @@ -154,18 +155,25 @@ # r6 == fastgil # XXX r7 == previous value of root_stack_top self.mc.gen_load_int(r.ip1.value, fastgil) - self.mc.LDREX(r.x0.value, r.r6.value) # load the lock value - self.mc.MOV_ri(r.ip.value, 1) - self.mc.CMP_ri(r.r3.value, 0) # is the lock free? - self.mc.STREX(r.r3.value, r.ip.value, r.r6.value, c=c.EQ) + self.mc.LDAXR(r.x1.value, r.ip1.value) # load the lock value + self.mc.MOVZ_r_u16(r.ip0.value, 1, 0) + self.mc.CMP_ri(r.x1.value, 0) # is the lock free? + if self.asm.cpu.gc_ll_descr.gcrootmap: + jump_val = XXX + else: + jump_val = 3 * 4 + self.mc.B_ofs_cond(jump_val, c.NE) + # jump over the next few instructions directly to the call + self.mc.STLXR(r.ip0.value, r.ip1.value, r.x1.value) # try to claim the lock - self.mc.CMP_ri(r.r3.value, 0, cond=c.EQ) # did this succeed? - if self.asm.cpu.cpuinfo.arch_version >= 7: - self.mc.DMB() + self.mc.CMP_wi(r.x1.value, 0) # did this succeed? + self.mc.DMB() # <- previous jump here + self.mc.B_ofs_cond((8 + 4)* 4, c.EQ) # jump over the call # the success of the lock acquisition is defined by # 'EQ is true', or equivalently by 'r3 == 0'. # if self.asm.cpu.gc_ll_descr.gcrootmap: + XXX # When doing a call_release_gil with shadowstack, there # is the risk that the 'rpy_fastgil' was free but the # current shadowstack can be the one of a different @@ -183,18 +191,15 @@ # by checking again r3. self.mc.CMP_ri(r.r3.value, 0) self.mc.STR_ri(r.r3.value, r.r6.value, cond=c.EQ) - else: - b1_location = self.mc.currpos() - self.mc.BKPT() # BEQ below # # save the result we just got - gpr_to_save, vfp_to_save = self.get_result_locs() - with saved_registers(self.mc, gpr_to_save, vfp_to_save): - self.mc.BL(self.asm.reacqgil_addr) - - # replace b1_location with B(here, c.EQ) - pmc = OverwritingBuilder(self.mc, b1_location, WORD) - pmc.B_offs(self.mc.currpos(), c.EQ) + self.mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD) + self.mc.STR_di(r.d0.value, r.sp.value, 0) + self.mc.STR_ri(r.x0.value, r.sp.value, WORD) + self.mc.BL(self.asm.reacqgil_addr) + self.mc.LDR_ri(r.d0.value, r.sp.value, 0) + self.mc.LDR_ri(r.x0.value, r.sp.value, WORD) + self.mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD) if not we_are_translated(): # for testing: now we can accesss self.mc.SUB_ri(r.fp.value, r.fp.value, 1) # fp again @@ -204,7 +209,7 @@ return [], [] if self.resloc.is_vfp_reg(): if self.restype == 'L': # long long - return [r.r0, r.r1], [] + return [r.r0], [] else: return [], [r.d0] assert self.resloc.is_core_reg() diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -379,6 +379,11 @@ assert 0 <= imm <= 4095 self.write32((base << 22) | (imm << 10) | (rn << 5) | 0b11111) + def CMP_wi(self, rn, imm): + base = 0b0111000100 + assert 0 <= imm <= 4095 + self.write32((base << 22) | (imm << 10) | (rn << 5) | 0b11111) + def CSET_r_flag(self, rd, cond): base = 0b10011010100 self.write32((base << 21) | (0b11111 << 16) | (cond << 12) | (1 << 10) | @@ -389,6 +394,14 @@ base = 0b11101010000 self.write32((base << 21) | (rm << 16) | (shift << 10) | (rn << 5) | 0b11111) + def LDAXR(self, rt, rn): + base = 0b1100100001011111111111 + self.write32((base << 10) | (rn << 5) | rt) + + def STLXR(self, rt, rn, rs): + base = 0b11001000000 + self.write32((base << 21) | (rs << 16) | (0b111111 << 10) | (rn << 5) | rt) + def NOP(self): self.write32(0b11010101000000110010000000011111) @@ -405,7 +418,7 @@ def B_ofs_cond(self, ofs, cond): base = 0b01010100 assert ofs & 0x3 == 0 - assert -1 << 10 < ofs < 1 << 10 + assert -1 << 21 < ofs < 1 << 21 imm = ofs >> 2 if imm < 0: xxx @@ -434,7 +447,7 @@ self.write32(0b11010100001 << 21) def DMB(self): - self.write32(0b1101010100000011001111110111111) + self.write32(0b11010101000000110011111110111111) def gen_load_int_full(self, r, value): self.MOVZ_r_u16(r, value & 0xFFFF, 0) diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -165,10 +165,14 @@ self.emit_int_comp_op(op, arglocs[0], arglocs[1]) return c.EQ + emit_comp_op_ptr_eq = emit_comp_op_int_eq + def emit_comp_op_int_ne(self, op, arglocs): self.emit_int_comp_op(op, arglocs[0], arglocs[1]) return c.NE + emit_comp_op_ptr_ne = emit_comp_op_int_ne + def emit_comp_op_uint_lt(self, op, arglocs): self.emit_int_comp_op(op, arglocs[0], arglocs[1]) return c.LO diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -417,6 +417,8 @@ prepare_comp_op_int_gt = prepare_int_cmp prepare_comp_op_int_ne = prepare_int_cmp prepare_comp_op_int_eq = prepare_int_cmp + prepare_comp_op_ptr_eq = prepare_int_cmp + prepare_comp_op_ptr_ne = prepare_int_cmp prepare_comp_op_uint_lt = prepare_int_cmp prepare_comp_op_uint_le = prepare_int_cmp prepare_comp_op_uint_ge = prepare_int_cmp From pypy.commits at gmail.com Tue Jun 25 13:44:27 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 10:44:27 -0700 (PDT) Subject: [pypy-commit] pypy arm64: a bit inefficient, but fix calling Message-ID: <5d125d7b.1c69fb81.26bdd.79a5@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96861:72e700c17875 Date: 2019-06-25 17:43 +0000 http://bitbucket.org/pypy/pypy/changeset/72e700c17875/ Log: a bit inefficient, but fix calling diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -927,6 +927,8 @@ elif loc.is_stack(): self.mc.LDR_ri(r.ip0.value, r.sp.value, pos) self.mc.STR_ri(r.ip0.value, r.fp.value, loc.value) + elif loc.is_vfp_reg(): + self.mc.LDR_di(loc.value, r.sp.value, pos) else: assert False, "wrong loc" @@ -943,7 +945,7 @@ if loc.is_stack(): self.mc.STR_di(prev_loc.value, r.fp.value, loc.value) elif loc.is_vfp_reg(): - self.mc.MOV_dd(prev_loc.value, loc.value) + self.mc.FMOV_dd(loc.value, prev_loc.value) else: assert False, "wrong loc" diff --git a/rpython/jit/backend/aarch64/callbuilder.py b/rpython/jit/backend/aarch64/callbuilder.py --- a/rpython/jit/backend/aarch64/callbuilder.py +++ b/rpython/jit/backend/aarch64/callbuilder.py @@ -38,19 +38,32 @@ non_float_regs.append(free_regs.pop()) else: stack_locs.append(arg) + + if stack_locs: + adj = len(stack_locs) + (len(stack_locs) & 1) + self.mc.SUB_ri(r.sp.value, r.sp.value, adj * WORD) + self.current_sp = adj + c = 0 + for loc in stack_locs: + self.asm.mov_loc_to_raw_stack(loc, c) + c += WORD + + move_back = False + if not self.fnloc.is_imm(): + if self.fnloc.is_core_reg(): + self.mc.MOV_rr(r.ip1.value, self.fnloc.value) + else: + assert self.fnloc.is_stack() + self.mc.LDR_ri(r.ip1.value, r.fp.value, self.fnloc.value) + self.fnloc = r.x8 + move_back = True + remap_frame_layout(self.asm, non_float_locs, non_float_regs, r.ip0) if float_locs: remap_frame_layout(self.asm, float_locs, float_regs, r.d8) - # move the remaining things to stack and adjust the stack - if not stack_locs: - return - adj = len(stack_locs) + (len(stack_locs) & 1) - self.mc.SUB_ri(r.sp.value, r.sp.value, adj * WORD) - self.current_sp = adj - c = 0 - for loc in stack_locs: - self.asm.mov_loc_to_raw_stack(loc, c) - c += WORD + + if move_back: + self.mc.MOV_rr(r.x8.value, r.ip1.value) def push_gcmap(self): noregs = self.asm.cpu.gc_ll_descr.is_shadow_stack() @@ -67,10 +80,10 @@ self.mc.BL(self.fnloc.value) return if self.fnloc.is_stack(): - self.mc.LDR_ri(r.ip0.value, r.fp.value, self.fnloc.value) - self.mc.BLR_r(r.ip0.value) + assert False, "we should never be here" else: assert self.fnloc.is_core_reg() + assert self.fnloc is r.x8 self.mc.BLR_r(self.fnloc.value) def restore_stack_pointer(self): @@ -197,7 +210,7 @@ self.mc.STR_di(r.d0.value, r.sp.value, 0) self.mc.STR_ri(r.x0.value, r.sp.value, WORD) self.mc.BL(self.asm.reacqgil_addr) - self.mc.LDR_ri(r.d0.value, r.sp.value, 0) + self.mc.LDR_di(r.d0.value, r.sp.value, 0) self.mc.LDR_ri(r.x0.value, r.sp.value, WORD) self.mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD) diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -72,11 +72,6 @@ def MOV_rr(self, rd, rn): self.ORR_rr(rd, r.xzr.value, rn) - def MOV_dd(self, rd, rn): - base = 0b00001110101 - self.write32((base << 21) | (rn << 16) | (0b000111 << 10) | (rn << 5) | - rd) - def UMOV_rd(self, rd, rn): base = 0b0100111000001000001111 self.write32((base << 10) | (rn << 5) | rd) @@ -224,6 +219,10 @@ base = 0b10001011000 | (s << 8) self.write32((base << 21) | (rm << 16) | (rn << 5) | (rd)) + def FMOV_dd(self, rd, rn): + base = 0b0001111001100000010000 + self.write32((base << 10) | (rn << 5) | rd) + def FADD_dd(self, rd, rn, rm): base = 0b00011110011 self.write32((base << 21) | (rm << 16) | (0b001010 << 10) | (rn << 5) | rd) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2987,11 +2987,19 @@ getter_name, primitive.cdecl(primitive.PrimitiveType[ARG], '*p'), var_name)) + c_source.append('#include ') c_source.append('') c_source.append('static void real%s(%s)' % ( fn_name, ', '.join(fn_args))) c_source.append('{') for i in range(len(ARGTYPES)): + if ARGTYPES[i] is lltype.Float: + c_source.append(' fprintf(stderr, "x%d = %%f\\n", x%d);' % (i, i)) + elif ARGTYPES[i] is lltype.Signed: + c_source.append(' fprintf(stderr, "x%d = %%ld\\n", x%d);' % (i, i)) + elif ARGTYPES[i] is rffi.UINT: + c_source.append(' fprintf(stderr, "x%d = %%u\\n", x%d);' % (i, i)) + for i in range(len(ARGTYPES)): c_source.append(' argcopy_%s_x%d = x%d;' % (fn_name, i, i)) c_source.append('}') c_source.append('RPY_EXPORTED void *%s(void)' % fn_name) From pypy.commits at gmail.com Tue Jun 25 14:41:21 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 11:41:21 -0700 (PDT) Subject: [pypy-commit] pypy arm64: typo Message-ID: <5d126ad1.1c69fb81.d6a33.697d@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96863:841338ea51d6 Date: 2019-06-25 18:36 +0000 http://bitbucket.org/pypy/pypy/changeset/841338ea51d6/ Log: typo diff --git a/rpython/jit/backend/aarch64/callbuilder.py b/rpython/jit/backend/aarch64/callbuilder.py --- a/rpython/jit/backend/aarch64/callbuilder.py +++ b/rpython/jit/backend/aarch64/callbuilder.py @@ -176,7 +176,7 @@ self.mc.LDR_ri(r.x11.value, r.sp.value, self.asm.saved_threadlocal_addr + self.current_sp) self.mc.LDR_ri(r.ip0.value, r.x11.value, p_errno) - self.mc.MOV_ri(r.x11.value, 0) + self.mc.MOVZ_r_u16(r.x11.value, 0, 0) self.mc.STR_ri(r.x11.value, r.ip0.value, 0) def read_real_errno(self, save_err): From pypy.commits at gmail.com Tue Jun 25 14:41:23 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 11:41:23 -0700 (PDT) Subject: [pypy-commit] pypy arm64: minor adjustments Message-ID: <5d126ad3.1c69fb81.c009f.d417@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96864:fe76a43ae509 Date: 2019-06-25 18:40 +0000 http://bitbucket.org/pypy/pypy/changeset/fe76a43ae509/ Log: minor adjustments diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -165,7 +165,7 @@ self.emit_int_comp_op(op, arglocs[0], arglocs[1]) return c.EQ - emit_comp_op_ptr_eq = emit_comp_op_int_eq + emit_comp_op_ptr_eq = emit_comp_op_instance_ptr_eq = emit_comp_op_int_eq def emit_comp_op_int_ne(self, op, arglocs): self.emit_int_comp_op(op, arglocs[0], arglocs[1]) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -417,7 +417,7 @@ prepare_comp_op_int_gt = prepare_int_cmp prepare_comp_op_int_ne = prepare_int_cmp prepare_comp_op_int_eq = prepare_int_cmp - prepare_comp_op_ptr_eq = prepare_int_cmp + prepare_comp_op_ptr_eq = prepare_comp_op_instance_ptr_eq = prepare_int_cmp prepare_comp_op_ptr_ne = prepare_int_cmp prepare_comp_op_uint_lt = prepare_int_cmp prepare_comp_op_uint_le = prepare_int_cmp From pypy.commits at gmail.com Tue Jun 25 14:44:43 2019 From: pypy.commits at gmail.com (fijal) Date: Tue, 25 Jun 2019 11:44:43 -0700 (PDT) Subject: [pypy-commit] pypy arm64: add passing test Message-ID: <5d126b9b.1c69fb81.d7a8b.db38@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96865:f8ea8c51ed28 Date: 2019-06-25 18:44 +0000 http://bitbucket.org/pypy/pypy/changeset/f8ea8c51ed28/ Log: add passing test diff --git a/rpython/jit/backend/aarch64/test/test_call.py b/rpython/jit/backend/aarch64/test/test_call.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/test/test_call.py @@ -0,0 +1,7 @@ +from rpython.jit.backend.aarch64.test.test_basic import JitAarch64Mixin +from rpython.jit.metainterp.test import test_call + +class TestCall(JitAarch64Mixin, test_call.CallTest): + # for the individual tests see + # ====> ../../../metainterp/test/test_call.py + pass From pypy.commits at gmail.com Wed Jun 26 07:16:17 2019 From: pypy.commits at gmail.com (mattip) Date: Wed, 26 Jun 2019 04:16:17 -0700 (PDT) Subject: [pypy-commit] pypy default: test, fix buffer_attach with a CBuffer. Alos add comment that except segfaults Message-ID: <5d135401.1c69fb81.9b033.5441@mx.google.com> Author: Matti Picus Branch: Changeset: r96866:cc5fc1bad809 Date: 2019-06-26 14:15 +0300 http://bitbucket.org/pypy/pypy/changeset/cc5fc1bad809/ Log: test, fix buffer_attach with a CBuffer. Alos add comment that except segfaults diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -5,6 +5,7 @@ cpython_api, Py_ssize_t, cpython_struct, bootstrap_function, slot_function, PyObjectFields, PyObject) from pypy.module.cpyext.pyobject import make_typedescr, decref, make_ref +from pypy.module.cpyext.buffer import CBuffer from pypy.module.array.interp_array import ArrayBuffer from pypy.objspace.std.bufferobject import W_Buffer @@ -33,7 +34,7 @@ def buffer_attach(space, py_obj, w_obj, w_userdata=None): """ - Fills a newly allocated PyBufferObject with the given (str) buffer object. + Fills a newly allocated PyBufferObject with the given buffer object. """ py_buf = rffi.cast(PyBufferObject, py_obj) py_buf.c_b_offset = 0 @@ -60,7 +61,17 @@ py_buf.c_b_base = make_ref(space, w_base) py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.w_array._charbuf_start()) py_buf.c_b_size = buf.getlength() + elif isinstance(buf, CBuffer): + py_buf.c_b_base = make_ref(space, buf.view.w_obj) + py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.view.ptr) + py_buf.c_b_size = buf.getlength() else: + # Raising in attach will segfault. + # It would be nice if we could handle the error more gracefully + # with something like this + # py_buf.c_b_base = lltype.nullptr(PyObject.TO) + # py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, 0) + # py_buf.c_b_size = buf.getlength() raise oefmt(space.w_NotImplementedError, "buffer flavor not supported") diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2573,6 +2573,11 @@ Py_RETURN_NONE; } +static PyObject * +passthrough(PyObject *self, PyObject* args) { + Py_INCREF(args); + return args; +} /*********************** Install Module **************************/ static PyMethodDef a_methods[] = { @@ -2584,6 +2589,7 @@ {"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL}, {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL}, {"subclass_with_attribute", (PyCFunction)subclass_with_attribute, METH_VARARGS, NULL}, + {"passthrough", (PyCFunction)passthrough, METH_O, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -79,6 +79,9 @@ assert str(buffer('a') + arr) == "a" + expected # python2 special cases empty-buffer + obj assert str(buffer('') + arr) == "array('i', [1, 2, 3, 4])" + # make sure buffer_attach is called + buf2 = module.passthrough(buf) + assert str(buf2) == str(buf) def test_releasebuffer(self): module = self.import_module(name='array') From pypy.commits at gmail.com Wed Jun 26 12:18:06 2019 From: pypy.commits at gmail.com (arigo) Date: Wed, 26 Jun 2019 09:18:06 -0700 (PDT) Subject: [pypy-commit] pypy default: Normalize encoding also when checking that it is "utf-8" after seeing a BOM Message-ID: <5d139abe.1c69fb81.e13f1.b728@mx.google.com> Author: Armin Rigo Branch: Changeset: r96867:9831511d5e3e Date: 2019-06-26 18:17 +0200 http://bitbucket.org/pypy/pypy/changeset/9831511d5e3e/ Log: Normalize encoding also when checking that it is "utf-8" after seeing a BOM diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -105,7 +105,7 @@ enc = 'utf-8' # If an encoding is explicitly given check that it is utf-8. decl_enc = _check_for_encoding(textsrc) - if decl_enc and decl_enc != "utf-8": + if decl_enc and _normalize_encoding(decl_enc) != "utf-8": raise error.SyntaxError("UTF-8 BOM with %s coding cookie" % decl_enc, filename=compile_info.filename) elif compile_info.flags & consts.PyCF_SOURCE_IS_UTF8: diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -56,6 +56,8 @@ input = "\xEF\xBB\xBF# coding: latin-1\nx" exc = py.test.raises(SyntaxError, self.parse, input).value assert exc.msg == "UTF-8 BOM with latin-1 coding cookie" + input = "\xEF\xBB\xBF# coding: UtF-8-yadda-YADDA\nx" + self.parse(input) # this does not raise input = "# coding: not-here" exc = py.test.raises(SyntaxError, self.parse, input).value assert exc.msg == "Unknown encoding: not-here" From pypy.commits at gmail.com Thu Jun 27 11:53:32 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 08:53:32 -0700 (PDT) Subject: [pypy-commit] pypy arm64: use x19 and x20 Message-ID: <5d14e67c.1c69fb81.d0d2f.5a70@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96870:b95baf585934 Date: 2019-06-27 15:06 +0000 http://bitbucket.org/pypy/pypy/changeset/b95baf585934/ Log: use x19 and x20 diff --git a/rpython/jit/backend/aarch64/TODO b/rpython/jit/backend/aarch64/TODO --- a/rpython/jit/backend/aarch64/TODO +++ b/rpython/jit/backend/aarch64/TODO @@ -2,3 +2,4 @@ * int_cmp - IMM * guard_nonnull_class - think about a better way * cond_call and following guard_exception +* we have a 3-register gap on JITFRAME diff --git a/rpython/jit/backend/aarch64/arch.py b/rpython/jit/backend/aarch64/arch.py --- a/rpython/jit/backend/aarch64/arch.py +++ b/rpython/jit/backend/aarch64/arch.py @@ -8,5 +8,5 @@ # A jitframe is a jit.backend.llsupport.llmodel.jitframe.JITFRAME # Stack frame fixed area # Currently only the force_index -JITFRAME_FIXED_SIZE = 16 + 8 -# 16 GPR + 8 VFP Regs # 20 if we want to use 4 extra x19..x22 +JITFRAME_FIXED_SIZE = 16 + 2 + 8 +# 18 GPR + 8 VFP Regs # 20 if we want to use 4 extra x19..x22 diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -405,7 +405,6 @@ mc.LDP_rri(r.x0.value, r.x1.value, r.sp.value, 0) mc.STR_ri(r.lr.value, r.sp.value, 0) - mc.STR_ri(r.x19.value, r.sp.value, WORD) # store the current gcmap(r0) in the jitframe gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') @@ -445,7 +444,6 @@ # return mc.LDR_ri(r.lr.value, r.sp.value, 0) - mc.LDR_ri(r.x19.value, r.sp.value, WORD) mc.ADD_ri(r.sp.value, r.sp.value, 2*WORD) mc.RET_r(r.lr.value) self._frame_realloc_slowpath = mc.materialize(self.cpu, []) diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -4,9 +4,17 @@ from rpython.jit.backend.aarch64.locations import RegisterLocation from rpython.jit.backend.aarch64 import registers as r from rpython.rlib.rarithmetic import intmask -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from rpython.tool.udir import udir +clear_cache = rffi.llexternal( + "__clear_cache", + [llmemory.Address, llmemory.Address], + lltype.Void, + _nowrapper=True, + sandboxsafe=True) + + class AbstractAarch64Builder(object): def write32(self, word): self.writechar(chr(word & 0xFF)) diff --git a/rpython/jit/backend/aarch64/registers.py b/rpython/jit/backend/aarch64/registers.py --- a/rpython/jit/backend/aarch64/registers.py +++ b/rpython/jit/backend/aarch64/registers.py @@ -11,7 +11,7 @@ vfpregisters = [VFPRegisterLocation(i) for i in range(32)] all_vfp_regs = vfpregisters[:8] -all_regs = registers[:16] #+ [x19, x20, x21, x22] +all_regs = registers[:16]+ [x19, x20] #, x21, x22] lr = x30 fp = x29 @@ -21,7 +21,7 @@ ip1 = x17 ip0 = x16 -callee_saved_registers = [] # x19, x20, x21, x22] +callee_saved_registers = [x19, x20] # , x21, x22] vfp_argument_regs = caller_vfp_resp = all_vfp_regs[:8] [d0, d1, d2, d3, d4, d5, d6, d7, d8, d9, d10, d11, d12, d13, d14, d15, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, @@ -30,4 +30,5 @@ vfp_ip = d15 argument_regs = [x0, x1, x2, x3, x4, x5, x6, x7] +callee_resp = [x19, x20] # ,x21, x22] caller_resp = argument_regs + [x8, x9, x10, x11, x12, x13, x14, x15] diff --git a/rpython/jit/backend/aarch64/runner.py b/rpython/jit/backend/aarch64/runner.py --- a/rpython/jit/backend/aarch64/runner.py +++ b/rpython/jit/backend/aarch64/runner.py @@ -10,7 +10,7 @@ """ARM 64""" backend_name = "aarch64" frame_reg = r.fp - all_reg_indexes = range(len(r.all_regs)) + all_reg_indexes = range(16) + [-1, -1, -1, 16, 17] gen_regs = r.all_regs float_regs = VFPRegisterManager.all_regs supports_floats = True From pypy.commits at gmail.com Thu Jun 27 11:53:34 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 08:53:34 -0700 (PDT) Subject: [pypy-commit] pypy arm64: start working on translation Message-ID: <5d14e67e.1c69fb81.f79ec.6cc7@mx.google.com> Author: fijal Branch: arm64 Changeset: r96871:28226a9abdd6 Date: 2019-06-27 17:52 +0200 http://bitbucket.org/pypy/pypy/changeset/28226a9abdd6/ Log: start working on translation diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -335,7 +335,6 @@ # It must keep stack alignment accordingly. mc = InstrBuilder() # - exc0 = exc1 = None mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD) mc.STR_ri(r.ip0.value, r.sp.value, WORD) mc.STR_ri(r.lr.value, r.sp.value, 0) @@ -349,10 +348,17 @@ # we're possibly called from the slowpath of malloc # save the caller saved registers # assuming we do not collect here - exc0, exc1 = r.r4, r.r5 - XXX - mc.PUSH([gpr.value for gpr in r.caller_resp] + [exc0.value, exc1.value]) - mc.VPUSH([vfpr.value for vfpr in r.caller_vfp_resp]) + exc0, exc1 = r.x19, r.x20 + mc.SUB_ri(r.sp.value, r.sp.value, (len(r.caller_resp) + 2 + len(r.caller_vfp_resp)) * WORD) + cur_stack = 0 + for i in range(0, len(r.caller_resp), 2): + mc.STP_rri(r.caller_resp[i].value, r.caller_resp[i + 1].value, r.sp.value, i * WORD) + cur_stack = len(r.caller_resp) + mc.STP_rri(exc0.value, exc1.value, cur_stack * WORD) + cur_stack += 2 + for i in range(len(r.caller_vfp_resp)): + mc.STR_di(r.caller_vfp_resp[i].value, r.sp.value, cur_stack * WORD) + cur_stack += 1 self._store_and_reset_exception(mc, exc0, exc1) mc.BL(func) @@ -360,13 +366,22 @@ if not for_frame: self._pop_all_regs_from_jitframe(mc, [], withfloats, callee_only=True) else: - XXX + exc0, exc1 = r.x19, r.x20 self._restore_exception(mc, exc0, exc1) - mc.VPOP([vfpr.value for vfpr in r.caller_vfp_resp]) + + cur_stack = 0 + for i in range(0, len(r.caller_resp), 2): + mc.LDP_rri(r.caller_resp[i].value, r.caller_resp[i + 1].value, r.sp.value, i * WORD) + cur_stack = len(r.caller_resp) + mc.LDP_rri(exc0.value, exc1.value, cur_stack * WORD) + cur_stack += 2 + for i in range(len(r.caller_vfp_resp)): + mc.LDR_di(r.caller_vfp_resp[i].value, r.sp.value, cur_stack * WORD) + cur_stack += 1 + assert exc0 is not None assert exc1 is not None - mc.POP([gpr.value for gpr in r.caller_resp] + - [exc0.value, exc1.value]) + # if withcards: # A final TEST8 before the RET, for the caller. Careful to @@ -430,10 +445,9 @@ gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: - xxx - self._load_shadowstack_top(mc, r.r5, gcrootmap) + self._load_shadowstack_top(mc, r.x19, gcrootmap) # store the new jitframe addr in the shadowstack - mc.STR_ri(r.x0.value, r.r5.value, imm=-WORD) + mc.STR_ri(r.x0.value, r.x19.value, -WORD) # reset the jf_gcmap field in the jitframe mc.gen_load_int(r.ip0.value, 0) @@ -600,14 +614,12 @@ def _reload_frame_if_necessary(self, mc): gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: - YYY rst = gcrootmap.get_root_stack_top_addr() - mc.gen_load_int(r.ip.value, rst) - self.load_reg(mc, r.ip, r.ip) - self.load_reg(mc, r.fp, r.ip, ofs=-WORD) + mc.gen_load_int(r.ip0.value, rst) + self.load_reg(mc, r.ip0, r.ip0) + self.load_reg(mc, r.fp, r.ip0, -WORD) wbdescr = self.cpu.gc_ll_descr.write_barrier_descr if gcrootmap and wbdescr: - YYY # frame never uses card marking, so we enforce this is not # an array self._write_barrier_fastpath(mc, wbdescr, [r.fp], array=False, @@ -680,8 +692,8 @@ size = self.mc.get_relative_pos() res = self.mc.materialize(self.cpu, allblocks, self.cpu.gc_ll_descr.gcrootmap) - #self.cpu.codemap.register_codemap( - # self.codemap.get_final_bytecode(res, size)) + self.cpu.codemap.register_codemap( + self.codemap.get_final_bytecode(res, size)) return res def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc): @@ -723,16 +735,17 @@ else: endaddr, lengthaddr, _ = self.cpu.insert_stack_check() # load stack end - self.mc.gen_load_int(r.ip.value, endaddr) # load ip, [end] - self.mc.LDR_ri(r.ip.value, r.ip.value) # LDR ip, ip + self.mc.gen_load_int(r.ip0.value, endaddr) # load ip0, [end] + self.mc.LDR_ri(r.ip0.value, r.ip0.value) # LDR ip0, ip0 # load stack length - self.mc.gen_load_int(r.lr.value, lengthaddr) # load lr, lengh - self.mc.LDR_ri(r.lr.value, r.lr.value) # ldr lr, *lengh + self.mc.gen_load_int(r.ip1.value, lengthaddr) # load ip1, lengh + self.mc.LDR_ri(r.ip1.value, r.ip1.value) # ldr ip1, *lengh # calculate ofs - self.mc.SUB_rr(r.ip.value, r.ip.value, r.sp.value) # SUB ip, current + self.mc.SUB_rr(r.ip0.value, r.ip0.value, r.sp.value) # SUB ip, current # if ofs - self.mc.CMP_rr(r.ip.value, r.lr.value) # CMP ip, lr - self.mc.BL(self.stack_check_slowpath, c=c.HI) # call if ip > lr + self.mc.CMP_rr(r.ip0.value, r.ip1.value) # CMP ip, lr + self.mc.B_ofs_cond(4 + 2, c.LS) + self.mc.B(self.stack_check_slowpath) # call if ip > lr def _call_header(self): stack_size = (len(r.callee_saved_registers) + 4) * WORD @@ -890,7 +903,7 @@ elif loc.is_stack(): self.mc.STR_ri(prev_loc.value, r.fp.value, loc.value) else: - XXX + assert False def _mov_imm_to_loc(self, prev_loc, loc): if loc.is_core_reg(): @@ -1025,19 +1038,18 @@ # mc.gen_load_int(r.ip1, ofs) # mc.STR_rr(source.value, base.value, r.ip1) - def load_reg(self, mc, target, base, ofs=0, helper=None): + def load_reg(self, mc, target, base, ofs=0, helper=r.ip0): if target.is_vfp_reg(): return self._load_vfp_reg(mc, target, base, ofs) elif target.is_core_reg(): - return self._load_core_reg(mc, target, base, ofs) + return self._load_core_reg(mc, target, base, ofs, helper) - def _load_core_reg(self, mc, target, base, ofs): + def _load_core_reg(self, mc, target, base, ofs, helper): if check_imm_arg(abs(ofs)): mc.LDR_ri(target.value, base.value, ofs) else: - XXX - mc.gen_load_int(helper.value, ofs, cond=cond) - mc.LDR_rr(target.value, base.value, helper.value, cond=cond) + mc.gen_load_int(helper.value, ofs) + mc.LDR_rr(target.value, base.value, helper.value) def check_frame_before_jump(self, target_token): if target_token in self.target_tokens_currently_compiling: diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -563,14 +563,17 @@ loc = r.ip0 else: assert v1.is_stack() - yyy + self.mc.LDR_ri(r.ip0.value, r.fp.value, v1.value) + loc = r.ip0 self.mc.CMP_rr(v0.value, loc.value) else: assert v0.is_vfp_reg() if v1.is_vfp_reg(): loc = v1 else: - xxx + assert v1.is_stack() + loc = r.vfp_ip + self.mc.LDR_di(r.vfp_ip.value, r.fp.value, v1.value) self.mc.FCMP_dd(v0.value, loc.value) self._emit_guard(op, c.EQ, arglocs[2:]) diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -579,28 +579,28 @@ effectinfo = calldescr.get_extra_info() if effectinfo is not None: oopspecindex = effectinfo.oopspecindex - if oopspecindex in (EffectInfo.OS_LLONG_ADD, - EffectInfo.OS_LLONG_SUB, - EffectInfo.OS_LLONG_AND, - EffectInfo.OS_LLONG_OR, - EffectInfo.OS_LLONG_XOR): - XXX - if self.cpu.cpuinfo.neon: - args = self._prepare_llong_binop_xx(op, fcond) - self.perform_extra(op, args, fcond) - return - elif oopspecindex == EffectInfo.OS_LLONG_TO_INT: - args = self._prepare_llong_to_int(op, fcond) - self.perform_extra(op, args, fcond) - return - elif oopspecindex == EffectInfo.OS_MATH_SQRT: + #if oopspecindex in (EffectInfo.OS_LLONG_ADD, + # EffectInfo.OS_LLONG_SUB, + # EffectInfo.OS_LLONG_AND, + # EffectInfo.OS_LLONG_OR, + # EffectInfo.OS_LLONG_XOR): + # XXX + # if self.cpu.cpuinfo.neon: + # args = self._prepare_llong_binop_xx(op, fcond) + # self.perform_extra(op, args, fcond) + # return + #elif oopspecindex == EffectInfo.OS_LLONG_TO_INT: + # args = self._prepare_llong_to_int(op, fcond) + # self.perform_extra(op, args, fcond) + # return + if oopspecindex == EffectInfo.OS_MATH_SQRT: args = self._prepare_op_math_sqrt(op) self.assembler.math_sqrt(op, args) return - elif oopspecindex == EffectInfo.OS_THREADLOCALREF_GET: - args = self._prepare_threadlocalref_get(op, fcond) - self.perform_extra(op, args, fcond) - return + #elif oopspecindex == EffectInfo.OS_THREADLOCALREF_GET: + # args = self._prepare_threadlocalref_get(op, fcond) + # self.perform_extra(op, args, fcond) + # return #elif oopspecindex == EffectInfo.OS_MATH_READ_TIMESTAMP: # ... return self._prepare_call(op) From pypy.commits at gmail.com Thu Jun 27 11:59:49 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 08:59:49 -0700 (PDT) Subject: [pypy-commit] pypy arm64: translation fixes Message-ID: <5d14e7f5.1c69fb81.97e4c.200f@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96872:73a46425439f Date: 2019-06-27 15:59 +0000 http://bitbucket.org/pypy/pypy/changeset/73a46425439f/ Log: translation fixes diff --git a/rpython/jit/backend/aarch64/TODO b/rpython/jit/backend/aarch64/TODO --- a/rpython/jit/backend/aarch64/TODO +++ b/rpython/jit/backend/aarch64/TODO @@ -2,4 +2,4 @@ * int_cmp - IMM * guard_nonnull_class - think about a better way * cond_call and following guard_exception -* we have a 3-register gap on JITFRAME +* stack check diff --git a/rpython/jit/backend/aarch64/codebuilder.py b/rpython/jit/backend/aarch64/codebuilder.py --- a/rpython/jit/backend/aarch64/codebuilder.py +++ b/rpython/jit/backend/aarch64/codebuilder.py @@ -427,8 +427,7 @@ assert ofs & 0x3 == 0 assert -1 << 21 < ofs < 1 << 21 imm = ofs >> 2 - if imm < 0: - xxx + assert imm > 0 # we seem not to need the - jump self.write32((base << 24) | (imm << 5) | cond) def B(self, target): diff --git a/rpython/jit/backend/aarch64/opassembler.py b/rpython/jit/backend/aarch64/opassembler.py --- a/rpython/jit/backend/aarch64/opassembler.py +++ b/rpython/jit/backend/aarch64/opassembler.py @@ -80,9 +80,11 @@ def emit_comp_op_int_add_ovf(self, op, arglocs): self.int_add_impl(op, arglocs, True) + return 0 def emit_comp_op_int_sub_ovf(self, op, arglocs): self.int_sub_impl(op, arglocs, True) + return 0 def emit_op_int_mul(self, op, arglocs): reg1, reg2, res = arglocs @@ -93,6 +95,7 @@ self.mc.SMULH_rr(r.ip0.value, reg1.value, reg2.value) self.mc.MUL_rr(res.value, reg1.value, reg2.value) self.mc.CMP_rr_shifted(r.ip0.value, res.value, 63) + return 0 def emit_op_int_and(self, op, arglocs): l0, l1, res = arglocs @@ -131,13 +134,6 @@ def emit_float_comp_op(self, op, l0, l1): self.mc.FCMP_dd(l0.value, l1.value) - emit_comp_op_float_lt = emit_float_comp_op - emit_comp_op_float_lt = emit_float_comp_op - emit_comp_op_float_lt = emit_float_comp_op - emit_comp_op_float_lt = emit_float_comp_op - emit_comp_op_float_lt = emit_float_comp_op - emit_comp_op_float_lt = emit_float_comp_op - emit_comp_op_float_lt = gen_float_comp_op_cc('float_lt', c.VFP_LT) emit_comp_op_float_le = gen_float_comp_op_cc('float_le', c.VFP_LE) emit_comp_op_float_eq = gen_float_comp_op_cc('float_eq', c.EQ) From pypy.commits at gmail.com Thu Jun 27 12:01:06 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 09:01:06 -0700 (PDT) Subject: [pypy-commit] pypy arm64: kill dead code Message-ID: <5d14e842.1c69fb81.3843b.3f19@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96873:eb09d690429b Date: 2019-06-27 16:00 +0000 http://bitbucket.org/pypy/pypy/changeset/eb09d690429b/ Log: kill dead code diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -579,20 +579,6 @@ effectinfo = calldescr.get_extra_info() if effectinfo is not None: oopspecindex = effectinfo.oopspecindex - #if oopspecindex in (EffectInfo.OS_LLONG_ADD, - # EffectInfo.OS_LLONG_SUB, - # EffectInfo.OS_LLONG_AND, - # EffectInfo.OS_LLONG_OR, - # EffectInfo.OS_LLONG_XOR): - # XXX - # if self.cpu.cpuinfo.neon: - # args = self._prepare_llong_binop_xx(op, fcond) - # self.perform_extra(op, args, fcond) - # return - #elif oopspecindex == EffectInfo.OS_LLONG_TO_INT: - # args = self._prepare_llong_to_int(op, fcond) - # self.perform_extra(op, args, fcond) - # return if oopspecindex == EffectInfo.OS_MATH_SQRT: args = self._prepare_op_math_sqrt(op) self.assembler.math_sqrt(op, args) From pypy.commits at gmail.com Thu Jun 27 12:59:42 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 09:59:42 -0700 (PDT) Subject: [pypy-commit] pypy arm64: hopefully finish translation Message-ID: <5d14f5fe.1c69fb81.a26ea.3004@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96874:3a7beb528024 Date: 2019-06-27 16:58 +0000 http://bitbucket.org/pypy/pypy/changeset/3a7beb528024/ Log: hopefully finish translation diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -692,8 +692,8 @@ size = self.mc.get_relative_pos() res = self.mc.materialize(self.cpu, allblocks, self.cpu.gc_ll_descr.gcrootmap) - self.cpu.codemap.register_codemap( - self.codemap.get_final_bytecode(res, size)) + #self.cpu.codemap.register_codemap( + # self.codemap.get_final_bytecode(res, size)) return res def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc): @@ -1039,12 +1039,7 @@ # mc.STR_rr(source.value, base.value, r.ip1) def load_reg(self, mc, target, base, ofs=0, helper=r.ip0): - if target.is_vfp_reg(): - return self._load_vfp_reg(mc, target, base, ofs) - elif target.is_core_reg(): - return self._load_core_reg(mc, target, base, ofs, helper) - - def _load_core_reg(self, mc, target, base, ofs, helper): + assert target.is_core_reg() if check_imm_arg(abs(ofs)): mc.LDR_ri(target.value, base.value, ofs) else: @@ -1101,7 +1096,7 @@ print "[ARM64/asm] %s not implemented" % op.getopname() raise NotImplementedError(op) -def notimplemented_guard_op(self, op, fcond, arglocs): +def notimplemented_guard_op(self, op, guard_op, fcond, arglocs): print "[ARM64/asm] %s not implemented" % op.getopname() raise NotImplementedError(op) diff --git a/rpython/jit/backend/aarch64/callbuilder.py b/rpython/jit/backend/aarch64/callbuilder.py --- a/rpython/jit/backend/aarch64/callbuilder.py +++ b/rpython/jit/backend/aarch64/callbuilder.py @@ -97,9 +97,11 @@ def load_result(self): resloc = self.resloc if self.restype == 'S': + assert False, "not supported yet" XXX self.mc.VMOV_sc(resloc.value, r.s0.value) elif self.restype == 'L': + assert False, "not possible on 64bit backend" YYY assert resloc.is_vfp_reg() self.mc.FMDRR(resloc.value, r.r0.value, r.r1.value) @@ -139,10 +141,9 @@ # Save this thread's shadowstack pointer into r7, for later comparison gcrootmap = self.asm.cpu.gc_ll_descr.gcrootmap if gcrootmap: - XXX rst = gcrootmap.get_root_stack_top_addr() - self.mc.gen_load_int(r.r5.value, rst) - self.mc.LDR_ri(r.r7.value, r.r5.value) + self.mc.gen_load_int(r.x19.value, rst) + self.mc.LDR_ri(r.x20.value, r.x19.value) # change 'rpy_fastgil' to 0 (it should be non-zero right now) self.mc.DMB() @@ -198,15 +199,14 @@ def move_real_result_and_call_reacqgil_addr(self, fastgil): # try to reacquire the lock. - # XXX r5 == &root_stack_top - # r6 == fastgil - # XXX r7 == previous value of root_stack_top + # x19 == &root_stack_top + # x20 == previous value of root_stack_top self.mc.gen_load_int(r.ip1.value, fastgil) self.mc.LDAXR(r.x1.value, r.ip1.value) # load the lock value self.mc.MOVZ_r_u16(r.ip0.value, 1, 0) self.mc.CMP_ri(r.x1.value, 0) # is the lock free? if self.asm.cpu.gc_ll_descr.gcrootmap: - jump_val = XXX + jump_val = 0 # XXX else: jump_val = 3 * 4 self.mc.B_ofs_cond(jump_val, c.NE) @@ -220,15 +220,16 @@ # 'EQ is true', or equivalently by 'r3 == 0'. # if self.asm.cpu.gc_ll_descr.gcrootmap: - XXX + raise Exception("not implemented yet") # When doing a call_release_gil with shadowstack, there # is the risk that the 'rpy_fastgil' was free but the # current shadowstack can be the one of a different # thread. So here we check if the shadowstack pointer # is still the same as before we released the GIL (saved - # in 'r7'), and if not, we fall back to 'reacqgil_addr'. - self.mc.LDR_ri(r.ip.value, r.r5.value, cond=c.EQ) - self.mc.CMP_rr(r.ip.value, r.r7.value, cond=c.EQ) + # in 'x20'), and if not, we fall back to 'reacqgil_addr'. + self.mc.LDR_ri(r.ip0.value, r.x19.value) + self.mc.CMP_rr(r.ip0.value, r.x20.value) + XXX b1_location = self.mc.currpos() self.mc.BKPT() # BEQ below # there are two cases here: either EQ was false from diff --git a/rpython/jit/backend/aarch64/regalloc.py b/rpython/jit/backend/aarch64/regalloc.py --- a/rpython/jit/backend/aarch64/regalloc.py +++ b/rpython/jit/backend/aarch64/regalloc.py @@ -814,7 +814,7 @@ l1 = self.loc(op.getarg(1)) imm_a1 = check_imm_box(op.getarg(1)) if not imm_a1: - l1 = self.make_sure_var_in_reg(op.getarg(1), [arg]) + l1 = self.make_sure_var_in_reg(op.getarg(1), [op.getarg(0)]) arglocs = self._guard_impl(op) return [arg, l1] + arglocs @@ -894,7 +894,7 @@ arg0 = self.make_sure_var_in_reg(args[0], args) arg1 = self.make_sure_var_in_reg(args[1], args) res = self.force_allocate_reg(op) - return [arg0, arg1, args[2], args[3], res] + return [arg0, arg1, imm(args[2].getint()), imm(args[3].getint()), res] def prepare_op_check_memory_error(self, op): argloc = self.make_sure_var_in_reg(op.getarg(0)) From pypy.commits at gmail.com Thu Jun 27 13:06:57 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 10:06:57 -0700 (PDT) Subject: [pypy-commit] pypy arm64: add some translation tests Message-ID: <5d14f7b1.1c69fb81.4f8d9.3469@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96875:143dcb62c9b7 Date: 2019-06-27 17:03 +0000 http://bitbucket.org/pypy/pypy/changeset/143dcb62c9b7/ Log: add some translation tests diff --git a/rpython/jit/backend/aarch64/test/test_zrpy_gc.py b/rpython/jit/backend/aarch64/test/test_zrpy_gc.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/test/test_zrpy_gc.py @@ -0,0 +1,6 @@ +from rpython.jit.backend.llsupport.test.zrpy_gc_test import CompileFrameworkTests + + +class TestShadowStack(CompileFrameworkTests): + gcrootfinder = "shadowstack" + gc = "incminimark" diff --git a/rpython/jit/backend/aarch64/test/test_zrpy_gc_boehm.py b/rpython/jit/backend/aarch64/test/test_zrpy_gc_boehm.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/test/test_zrpy_gc_boehm.py @@ -0,0 +1,1 @@ +from rpython.jit.backend.llsupport.test.zrpy_gc_boehm_test import compile_boehm_test as test_compile_boehm diff --git a/rpython/jit/backend/aarch64/test/test_ztranslation_call_assembler.py b/rpython/jit/backend/aarch64/test/test_ztranslation_call_assembler.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/test/test_ztranslation_call_assembler.py @@ -0,0 +1,23 @@ +from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationTestCallAssembler +from rpython.translator.translator import TranslationContext +from rpython.config.translationoption import DEFL_GC +from rpython.jit.backend.x86.arch import WORD +import sys + + +# On Windows, this test crashes obscurely, but only if compiled with +# Boehm, not if run with no GC at all. So for now we'll assume it is +# really a Boehm bug, or maybe a Boehm-on-Windows-specific issue, and +# skip. +if sys.platform == 'win32': + import py + py.test.skip("crashes on Windows (Boehm issue?)") + + +class TestTranslationCallAssemblerX86(TranslationTestCallAssembler): + def _check_cbuilder(self, cbuilder): + #We assume here that we have sse2. If not, the CPUClass + # needs to be changed to CPU386_NO_SSE2, but well. + if WORD == 4 and sys.platform != 'win32': + assert '-msse2' in cbuilder.eci.compile_extra + assert '-mfpmath=sse' in cbuilder.eci.compile_extra diff --git a/rpython/jit/backend/aarch64/test/test_ztranslation_external_exception.py b/rpython/jit/backend/aarch64/test/test_ztranslation_external_exception.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/test/test_ztranslation_external_exception.py @@ -0,0 +1,19 @@ +from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationRemoveTypePtrTest +from rpython.translator.translator import TranslationContext +from rpython.config.translationoption import DEFL_GC +from rpython.translator.platform import platform as compiler + +if compiler.name == 'msvc': + _MSVC = True +else: + _MSVC = False + +class TestTranslationRemoveTypePtrX86(TranslationRemoveTypePtrTest): + def _get_TranslationContext(self): + t = TranslationContext() + t.config.translation.gc = DEFL_GC # 'hybrid' or 'minimark' + if not _MSVC: + t.config.translation.gcrootfinder = 'asmgcc' + t.config.translation.list_comprehension_operations = True + t.config.translation.gcremovetypeptr = True + return t diff --git a/rpython/jit/backend/aarch64/test/test_ztranslation_jit_stats.py b/rpython/jit/backend/aarch64/test/test_ztranslation_jit_stats.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/aarch64/test/test_ztranslation_jit_stats.py @@ -0,0 +1,14 @@ +from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationTestJITStats +from rpython.translator.translator import TranslationContext +from rpython.config.translationoption import DEFL_GC +from rpython.jit.backend.x86.arch import WORD +import sys + + +class TestTranslationJITStatsX86(TranslationTestJITStats): + def _check_cbuilder(self, cbuilder): + #We assume here that we have sse2. If not, the CPUClass + # needs to be changed to CPU386_NO_SSE2, but well. + if WORD == 4 and sys.platform != 'win32': + assert '-msse2' in cbuilder.eci.compile_extra + assert '-mfpmath=sse' in cbuilder.eci.compile_extra From pypy.commits at gmail.com Thu Jun 27 13:06:59 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 10:06:59 -0700 (PDT) Subject: [pypy-commit] pypy arm64: fix tests Message-ID: <5d14f7b3.1c69fb81.73c3a.921d@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96876:776401f19133 Date: 2019-06-27 17:06 +0000 http://bitbucket.org/pypy/pypy/changeset/776401f19133/ Log: fix tests diff --git a/rpython/jit/backend/aarch64/test/test_ztranslation_call_assembler.py b/rpython/jit/backend/aarch64/test/test_ztranslation_call_assembler.py --- a/rpython/jit/backend/aarch64/test/test_ztranslation_call_assembler.py +++ b/rpython/jit/backend/aarch64/test/test_ztranslation_call_assembler.py @@ -1,7 +1,4 @@ from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationTestCallAssembler -from rpython.translator.translator import TranslationContext -from rpython.config.translationoption import DEFL_GC -from rpython.jit.backend.x86.arch import WORD import sys @@ -14,10 +11,5 @@ py.test.skip("crashes on Windows (Boehm issue?)") -class TestTranslationCallAssemblerX86(TranslationTestCallAssembler): - def _check_cbuilder(self, cbuilder): - #We assume here that we have sse2. If not, the CPUClass - # needs to be changed to CPU386_NO_SSE2, but well. - if WORD == 4 and sys.platform != 'win32': - assert '-msse2' in cbuilder.eci.compile_extra - assert '-mfpmath=sse' in cbuilder.eci.compile_extra +class TestTranslationCallAssemblerAarch64(TranslationTestCallAssembler): + pass \ No newline at end of file diff --git a/rpython/jit/backend/aarch64/test/test_ztranslation_external_exception.py b/rpython/jit/backend/aarch64/test/test_ztranslation_external_exception.py --- a/rpython/jit/backend/aarch64/test/test_ztranslation_external_exception.py +++ b/rpython/jit/backend/aarch64/test/test_ztranslation_external_exception.py @@ -1,19 +1,9 @@ from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationRemoveTypePtrTest from rpython.translator.translator import TranslationContext -from rpython.config.translationoption import DEFL_GC -from rpython.translator.platform import platform as compiler -if compiler.name == 'msvc': - _MSVC = True -else: - _MSVC = False - -class TestTranslationRemoveTypePtrX86(TranslationRemoveTypePtrTest): +class TestTranslationRemoveTypePtrAarch64(TranslationRemoveTypePtrTest): def _get_TranslationContext(self): t = TranslationContext() - t.config.translation.gc = DEFL_GC # 'hybrid' or 'minimark' - if not _MSVC: - t.config.translation.gcrootfinder = 'asmgcc' t.config.translation.list_comprehension_operations = True t.config.translation.gcremovetypeptr = True return t diff --git a/rpython/jit/backend/aarch64/test/test_ztranslation_jit_stats.py b/rpython/jit/backend/aarch64/test/test_ztranslation_jit_stats.py --- a/rpython/jit/backend/aarch64/test/test_ztranslation_jit_stats.py +++ b/rpython/jit/backend/aarch64/test/test_ztranslation_jit_stats.py @@ -1,14 +1,5 @@ from rpython.jit.backend.llsupport.test.ztranslation_test import TranslationTestJITStats -from rpython.translator.translator import TranslationContext -from rpython.config.translationoption import DEFL_GC -from rpython.jit.backend.x86.arch import WORD -import sys -class TestTranslationJITStatsX86(TranslationTestJITStats): - def _check_cbuilder(self, cbuilder): - #We assume here that we have sse2. If not, the CPUClass - # needs to be changed to CPU386_NO_SSE2, but well. - if WORD == 4 and sys.platform != 'win32': - assert '-msse2' in cbuilder.eci.compile_extra - assert '-mfpmath=sse' in cbuilder.eci.compile_extra +class TestTranslationJITStatsAarch64(TranslationTestJITStats): + pass \ No newline at end of file From pypy.commits at gmail.com Thu Jun 27 13:11:07 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 10:11:07 -0700 (PDT) Subject: [pypy-commit] pypy arm64: gen_footer_shadowstack Message-ID: <5d14f8ab.1c69fb81.66623.99af@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96877:d15f9ba0be63 Date: 2019-06-27 17:10 +0000 http://bitbucket.org/pypy/pypy/changeset/d15f9ba0be63/ Log: gen_footer_shadowstack diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -1015,6 +1015,13 @@ mc.RET_r(r.lr.value) + def gen_footer_shadowstack(self, gcrootmap, mc): + rst = gcrootmap.get_root_stack_top_addr() + mc.gen_load_int(r.ip0.value, rst) + self.load_reg(mc, r.ip1, r.ip0) + mc.SUB_ri(r.ip1.value, r.ip1.value, WORD) + self.store_reg(mc, r.ip1, r.ip0) + def store_reg(self, mc, source, base, ofs=0, helper=None): # uses r.ip1 as a temporary if source.is_vfp_reg(): From pypy.commits at gmail.com Thu Jun 27 13:37:08 2019 From: pypy.commits at gmail.com (fijal) Date: Thu, 27 Jun 2019 10:37:08 -0700 (PDT) Subject: [pypy-commit] pypy arm64: missing bits Message-ID: <5d14fec4.1c69fb81.492e0.f66c@mx.google.com> Author: Maciej Fijalkowski Branch: arm64 Changeset: r96878:d12d1f105c66 Date: 2019-06-27 17:36 +0000 http://bitbucket.org/pypy/pypy/changeset/d12d1f105c66/ Log: missing bits diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -354,7 +354,7 @@ for i in range(0, len(r.caller_resp), 2): mc.STP_rri(r.caller_resp[i].value, r.caller_resp[i + 1].value, r.sp.value, i * WORD) cur_stack = len(r.caller_resp) - mc.STP_rri(exc0.value, exc1.value, cur_stack * WORD) + mc.STP_rri(exc0.value, exc1.value, r.sp.value, cur_stack * WORD) cur_stack += 2 for i in range(len(r.caller_vfp_resp)): mc.STR_di(r.caller_vfp_resp[i].value, r.sp.value, cur_stack * WORD) @@ -373,7 +373,7 @@ for i in range(0, len(r.caller_resp), 2): mc.LDP_rri(r.caller_resp[i].value, r.caller_resp[i + 1].value, r.sp.value, i * WORD) cur_stack = len(r.caller_resp) - mc.LDP_rri(exc0.value, exc1.value, cur_stack * WORD) + mc.LDP_rri(exc0.value, exc1.value, r.sp.value, cur_stack * WORD) cur_stack += 2 for i in range(len(r.caller_vfp_resp)): mc.LDR_di(r.caller_vfp_resp[i].value, r.sp.value, cur_stack * WORD) @@ -462,6 +462,12 @@ mc.RET_r(r.lr.value) self._frame_realloc_slowpath = mc.materialize(self.cpu, []) + def _load_shadowstack_top(self, mc, reg, gcrootmap): + rst = gcrootmap.get_root_stack_top_addr() + mc.gen_load_int(reg.value, rst) + self.load_reg(mc, reg, reg) + return rst + def _store_and_reset_exception(self, mc, excvalloc=None, exctploc=None, on_frame=False): """ Resest the exception. If excvalloc is None, then store it on the From pypy.commits at gmail.com Fri Jun 28 14:49:53 2019 From: pypy.commits at gmail.com (arigo) Date: Fri, 28 Jun 2019 11:49:53 -0700 (PDT) Subject: [pypy-commit] pypy arm64: Write gen_shadowstack_header() from arm at revision ed056c08b76f Message-ID: <5d166151.1c69fb81.30350.1abd@mx.google.com> Author: Armin Rigo Branch: arm64 Changeset: r96880:99b712dabafd Date: 2019-06-28 20:48 +0200 http://bitbucket.org/pypy/pypy/changeset/99b712dabafd/ Log: Write gen_shadowstack_header() from arm at revision ed056c08b76f diff --git a/rpython/jit/backend/aarch64/assembler.py b/rpython/jit/backend/aarch64/assembler.py --- a/rpython/jit/backend/aarch64/assembler.py +++ b/rpython/jit/backend/aarch64/assembler.py @@ -1021,11 +1021,27 @@ mc.RET_r(r.lr.value) + def gen_shadowstack_header(self, gcrootmap): + # we push two words, like the x86 backend does: + # the '1' is to benefit from the shadowstack 'is_minor' optimization + rst = gcrootmap.get_root_stack_top_addr() + self.mc.gen_load_int(r.ip1.value, rst) + # x8 = *ip1 + self.load_reg(self.mc, r.x8, r.ip1) + # x8[0] = 1 + self.mc.gen_load_int(r.ip0.value, 1) + self.store_reg(self.mc, r.ip0, r.x8) + # x8[1] = r.fp + self.store_reg(self.mc, r.fp, r.x8, WORD) + # *ip1 = x8 + 2 * WORD + self.mc.ADD_ri(r.x8.value, r.x8.value, 2 * WORD) + self.store_reg(self.mc, r.x8, r.ip1) + def gen_footer_shadowstack(self, gcrootmap, mc): rst = gcrootmap.get_root_stack_top_addr() mc.gen_load_int(r.ip0.value, rst) self.load_reg(mc, r.ip1, r.ip0) - mc.SUB_ri(r.ip1.value, r.ip1.value, WORD) + mc.SUB_ri(r.ip1.value, r.ip1.value, 2 * WORD) # two words, see above self.store_reg(mc, r.ip1, r.ip0) def store_reg(self, mc, source, base, ofs=0, helper=None): From pypy.commits at gmail.com Sat Jun 29 04:03:53 2019 From: pypy.commits at gmail.com (arigo) Date: Sat, 29 Jun 2019 01:03:53 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: #3033 Test and fix for float.__round__(None) Message-ID: <5d171b69.1c69fb81.4fbdf.7be3@mx.google.com> Author: Armin Rigo Branch: py3.6 Changeset: r96881:d985f090d5c1 Date: 2019-06-29 10:03 +0200 http://bitbucket.org/pypy/pypy/changeset/d985f090d5c1/ Log: #3033 Test and fix for float.__round__(None) diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -929,7 +929,7 @@ # Algorithm copied directly from CPython x = w_float.floatval - if w_ndigits is None: + if space.is_none(w_ndigits): # single-argument round: round to nearest integer rounded = rfloat.round_away(x) if math.fabs(x - rounded) == 0.5: diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -156,7 +156,7 @@ # efficient for W_IntObject from pypy.objspace.std.longobject import newlong - if w_ndigits is None: + if space.is_none(w_ndigits): return self.int(space) ndigits = space.bigint_w(space.index(w_ndigits)) diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py --- a/pypy/objspace/std/test/test_floatobject.py +++ b/pypy/objspace/std/test/test_floatobject.py @@ -324,6 +324,7 @@ assert -2.0 == round(-1.5) assert -2.0 == round(-1.5, 0) assert -2.0 == round(-1.5, 0) + assert -2.0 == round(-1.5, None) assert 22.2 == round(22.222222, 1) assert 20.0 == round(22.22222, -1) assert 0.0 == round(22.22222, -2) @@ -334,6 +335,11 @@ assert math.copysign(1., round(-123.456, -700)) == -1. assert round(2.5, 0) == 2.0 + def test_round_special_method(self): + assert 2.0 == 1.9 .__round__() + assert -2.0 == -1.5 .__round__(None) + assert 20.0 == 22.22222 .__round__(-1) + def test_special_float_method(self): class a(object): def __float__(self): diff --git a/pypy/objspace/std/test/test_intobject.py b/pypy/objspace/std/test/test_intobject.py --- a/pypy/objspace/std/test/test_intobject.py +++ b/pypy/objspace/std/test/test_intobject.py @@ -743,6 +743,14 @@ x = -b assert x.__rsub__(2) == (2 + b) + def test_round_special_method(self): + assert 567 .__round__(-1) == 570 + assert 567 .__round__() == 567 + import sys + if '__pypy__' in sys.builtin_module_names: + assert 567 .__round__(None) == 567 # fails on CPython + + class AppTestIntShortcut(AppTestInt): spaceconfig = {"objspace.std.intshortcut": True} From pypy.commits at gmail.com Sat Jun 29 12:36:41 2019 From: pypy.commits at gmail.com (yodada) Date: Sat, 29 Jun 2019 09:36:41 -0700 (PDT) Subject: [pypy-commit] pypy record-exact-value: [yodada] Message-ID: <5d179399.1c69fb81.49cd9.c5b0@mx.google.com> Author: Lin Cheng Branch: record-exact-value Changeset: r96882:a1ad69dfd5ab Date: 2019-05-21 17:41 -0400 http://bitbucket.org/pypy/pypy/changeset/a1ad69dfd5ab/ Log: [yodada] Ported type freezing into pypy mapdict. Still debugging... diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -12,6 +12,15 @@ from pypy.objspace.std.typeobject import MutableCell +class UninitedType(W_Root): + pass + +class MutatedType(W_Root): + pass + +w_uninited_type = UninitedType() +w_mutated_type = MutatedType() + erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage item") erase_map, unerase_map = rerased.new_erasing_pair("map") erase_list, unerase_list = rerased.new_erasing_pair("mapdict storage list") @@ -40,6 +49,16 @@ assert isinstance(terminator, Terminator) self.terminator = terminator + def record_type_info(self, w_value): + # if already mutated, no need to record + if self.value_type is w_mutated_type.__class__: + return + if self.value_type is UninitedType: + self.value_type = w_value.__class__ # record type info + else: + if self.value_type is not w_value.__class__: + self.value_type = w_mutated_type.__class__ # type mutated + def read(self, obj, name, index): attr = self.find_map_attr(name, index) if attr is None: @@ -51,7 +70,12 @@ ): return self._pure_mapdict_read_storage(obj, attr.storageindex) else: - return obj._mapdict_read_storage(attr.storageindex) + value = obj._mapdict_read_storage(attr.storageindex) + value_type = attr.value_type + if value_type is not w_mutated_type.__class__ and value_type is not UninitedType: + assert value is not None + jit.record_exact_class(value, value_type) + return value @jit.elidable def _pure_mapdict_read_storage(self, obj, storageindex): @@ -63,6 +87,7 @@ return self.terminator._write_terminator(obj, name, index, w_value) if not attr.ever_mutated: attr.ever_mutated = True + attr.record_type_info(w_value) obj._mapdict_write_storage(attr.storageindex, w_value) return True @@ -242,6 +267,7 @@ stack_index += 2 current = current.back attr._switch_map_and_write_storage(obj, w_value) + attr.record_type_info(w_value) if not stack_index: return @@ -374,7 +400,7 @@ return Terminator.set_terminator(self, obj, terminator) class PlainAttribute(AbstractAttribute): - _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated?', 'order'] + _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated?', 'order', 'value_type?'] def __init__(self, name, index, back): AbstractAttribute.__init__(self, back.space, back.terminator) @@ -385,6 +411,7 @@ self._size_estimate = self.length() * NUM_DIGITS_POW2 self.ever_mutated = False self.order = len(back.cache_attrs) if back.cache_attrs else 0 + self.value_type = UninitedType def _copy_attr(self, obj, new_obj): w_value = self.read(obj, self.name, self.index) From pypy.commits at gmail.com Sat Jun 29 12:36:42 2019 From: pypy.commits at gmail.com (yodada) Date: Sat, 29 Jun 2019 09:36:42 -0700 (PDT) Subject: [pypy-commit] pypy record-exact-value: ( yodada ) Message-ID: <5d17939a.1c69fb81.f8686.604c@mx.google.com> Author: Lin Cheng Branch: record-exact-value Changeset: r96883:5d48dd8c5279 Date: 2019-05-27 10:15 -0400 http://bitbucket.org/pypy/pypy/changeset/5d48dd8c5279/ Log: ( yodada ) Added tests related to the value_type in PlainAttribute class. Note that Terminator's does not need to have value_type field. diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -529,6 +529,47 @@ obj.setdictvalue(space, a, 50) assert c.terminator.size_estimate() in [(i + 10) // 2, (i + 11) // 2] +def test_value_type(): + c = Class() + obj = c.instantiate() + obj.setdictvalue(space, "a", 10) + # Added an integer, so the outter most attribute + # has type 'int' + assert obj.map.value_type is type(10) + obj.setdictvalue(space, "a", "cornell") + # Since we are storing both a string and an int into + # slot "a", the value_type should be mutated_type + assert obj.map.value_type is w_mutated_type.__class__ + +def test_value_type_reordering(): + c = Class() + obj1 = c.instantiate() + obj2 = c.instantiate() + + obj1.setdictvalue(space, "a", 10) + obj1.setdictvalue(space, "b", 10) + # now obj1's map is + # < b: int < a: int < terminal + obj2.setdictvalue(space, "b", "cornell") + # now obj2's map is + # < b: string < terminal + # Both of them should have well-behaving value_type + assert obj1.map.value_type is type(10) + assert obj1.map.back.value_type is type(10) + assert obj2.map.value_type is type("cornell") + assert obj1.map.terminator is obj2.map.terminator + + # Now we add an "a" attribute into obj2, then during + # reordering, the type mismatch should be caught + obj2.setdictvalue(space, "a", 10) + assert obj1.map.value_type is w_mutated_type.__class__ + assert obj2.map.value_type is w_mutated_type.__class__ + assert obj1.map is obj2.map + # However, the "a" attribute should be intact + assert obj1.map.back is obj2.map.back + assert obj1.map.back.value_type is type(10) + assert obj2.map.back.value_type is type(10) + # ___________________________________________________________ # dict tests From pypy.commits at gmail.com Sat Jun 29 12:36:44 2019 From: pypy.commits at gmail.com (yodada) Date: Sat, 29 Jun 2019 09:36:44 -0700 (PDT) Subject: [pypy-commit] pypy record-exact-value: ( yodada ) Message-ID: <5d17939c.1c69fb81.1e4d3.df29@mx.google.com> Author: Lin Cheng Branch: record-exact-value Changeset: r96884:ab1447e95a0c Date: 2019-05-27 10:17 -0400 http://bitbucket.org/pypy/pypy/changeset/ab1447e95a0c/ Log: ( yodada ) Moved value_type init from PlainAttribute to AbstractAttribute. Just in case there are other accesses to this field when the Attribute is not a Plain one. diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -40,7 +40,7 @@ class AbstractAttribute(object): - _immutable_fields_ = ['terminator'] + _immutable_fields_ = ['terminator', 'value_type?'] cache_attrs = None _size_estimate = 0 @@ -48,6 +48,7 @@ self.space = space assert isinstance(terminator, Terminator) self.terminator = terminator + self.value_type = UninitedType def record_type_info(self, w_value): # if already mutated, no need to record @@ -400,7 +401,7 @@ return Terminator.set_terminator(self, obj, terminator) class PlainAttribute(AbstractAttribute): - _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated?', 'order', 'value_type?'] + _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated?', 'order'] def __init__(self, name, index, back): AbstractAttribute.__init__(self, back.space, back.terminator) @@ -411,7 +412,6 @@ self._size_estimate = self.length() * NUM_DIGITS_POW2 self.ever_mutated = False self.order = len(back.cache_attrs) if back.cache_attrs else 0 - self.value_type = UninitedType def _copy_attr(self, obj, new_obj): w_value = self.read(obj, self.name, self.index) From pypy.commits at gmail.com Sat Jun 29 12:36:46 2019 From: pypy.commits at gmail.com (yodada) Date: Sat, 29 Jun 2019 09:36:46 -0700 (PDT) Subject: [pypy-commit] pypy record-exact-value: ( yodada ) Message-ID: <5d17939e.1c69fb81.afee9.bd3a@mx.google.com> Author: Lin Cheng Branch: record-exact-value Changeset: r96885:b30cfe316af0 Date: 2019-05-27 11:10 -0400 http://bitbucket.org/pypy/pypy/changeset/b30cfe316af0/ Log: ( yodada ) Added pypy-c to .hgignore diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -92,4 +92,4 @@ .hypothesis/ ^release/ ^rpython/_cache$ - +pypy-c From pypy.commits at gmail.com Sat Jun 29 12:36:48 2019 From: pypy.commits at gmail.com (yodada) Date: Sat, 29 Jun 2019 09:36:48 -0700 (PDT) Subject: [pypy-commit] pypy record-exact-value: ( yodada ) Message-ID: <5d1793a0.1c69fb81.8bb8a.c433@mx.google.com> Author: Lin Cheng Branch: record-exact-value Changeset: r96886:664016ff8cc1 Date: 2019-05-27 22:43 -0400 http://bitbucket.org/pypy/pypy/changeset/664016ff8cc1/ Log: ( yodada ) Added terminal map into pypy's mapdict. It works but may not be 100% correct diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -2,6 +2,7 @@ from rpython.rlib import jit, objectmodel, debug, rerased from rpython.rlib.rarithmetic import intmask, r_uint +from rpython.rlib.objectmodel import instantiate from pypy.interpreter.baseobjspace import W_Root from pypy.objspace.std.dictmultiobject import ( @@ -10,6 +11,7 @@ W_DictObject, BytesDictStrategy, UnicodeDictStrategy ) from pypy.objspace.std.typeobject import MutableCell +from pypy.objspace.std.objectobject import W_ObjectObject class UninitedType(W_Root): @@ -38,17 +40,33 @@ # dict) LIMIT_MAP_ATTRIBUTES = 80 +#class AbstractAttribute_MetaBase(object): +# _immutable_fields_ = ['is_terminal?'] +# +# def __init__(self): +# self.is_terminal = True +# +#class UninitedMap(AbstractAttribute_MetaBase): +# pass +# +#class MutatedMap(AbstractAttribute_MetaBase): +# pass +# +#uninited_map = instantiate(UninitedMap) +#mutated_map = instantiate(MutatedMap) class AbstractAttribute(object): - _immutable_fields_ = ['terminator', 'value_type?'] + _immutable_fields_ = ['terminator', 'value_type?', 'value_map?', 'is_terminal?'] cache_attrs = None _size_estimate = 0 def __init__(self, space, terminator): + #AbstractAttribute_MetaBase.__init__(self) self.space = space assert isinstance(terminator, Terminator) self.terminator = terminator self.value_type = UninitedType + self.is_terminal = True def record_type_info(self, w_value): # if already mutated, no need to record @@ -59,6 +77,17 @@ else: if self.value_type is not w_value.__class__: self.value_type = w_mutated_type.__class__ # type mutated + return + # if value_map has already mutated, no need to record + if self.value_map is mutated_map: + return + if isinstance(w_value, W_ObjectObject): + if self.value_map is uninited_map: + self.value_map = w_value._get_mapdict_map() + else: + if self.value_map is not w_value._get_mapdict_map(): + self.value_map = mutated_map + return def read(self, obj, name, index): attr = self.find_map_attr(name, index) @@ -76,6 +105,10 @@ if value_type is not w_mutated_type.__class__ and value_type is not UninitedType: assert value is not None jit.record_exact_class(value, value_type) + if isinstance(value, W_ObjectObject): + inner_map = attr.value_map + if inner_map is not mutated_map and inner_map is not uninited_map and inner_map.is_terminal: + jit.record_exact_value(value.map, inner_map) return value @jit.elidable @@ -85,6 +118,8 @@ def write(self, obj, name, index, w_value): attr = self.find_map_attr(name, index) if attr is None: + if self.is_terminal: + self.is_terminal = False return self.terminator._write_terminator(obj, name, index, w_value) if not attr.ever_mutated: attr.ever_mutated = True @@ -293,6 +328,8 @@ def __repr__(self): return "<%s>" % (self.__class__.__name__,) +uninited_map = instantiate(AbstractAttribute) +mutated_map = instantiate(AbstractAttribute) class Terminator(AbstractAttribute): _immutable_fields_ = ['w_cls'] @@ -412,6 +449,7 @@ self._size_estimate = self.length() * NUM_DIGITS_POW2 self.ever_mutated = False self.order = len(back.cache_attrs) if back.cache_attrs else 0 + self.value_map = uninited_map def _copy_attr(self, obj, new_obj): w_value = self.read(obj, self.name, self.index)