From pypy.commits at gmail.com Mon Oct 1 10:42:26 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 01 Oct 2018 07:42:26 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Resync boolobject.h with CPython, rename _Py_ZeroStruct to _Py_FalseStruct Message-ID: <5bb23252.1c69fb81.a19b8.dc98@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r95174:f4bc915225bc Date: 2018-10-01 15:41 +0100 http://bitbucket.org/pypy/pypy/changeset/f4bc915225bc/ Log: Resync boolobject.h with CPython, rename _Py_ZeroStruct to _Py_FalseStruct diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -668,7 +668,7 @@ 'PyObject*', 'space.w_None', header=pypy_decl) register_global('_Py_TrueStruct', 'PyObject*', 'space.w_True', header=pypy_decl) -register_global('_Py_ZeroStruct', +register_global('_Py_FalseStruct', 'PyObject*', 'space.w_False', header=pypy_decl) register_global('_Py_NotImplementedStruct', 'PyObject*', 'space.w_NotImplemented', header=pypy_decl) diff --git a/pypy/module/cpyext/include/boolobject.h b/pypy/module/cpyext/include/boolobject.h --- a/pypy/module/cpyext/include/boolobject.h +++ b/pypy/module/cpyext/include/boolobject.h @@ -1,5 +1,4 @@ - -/* Bool object interface */ +/* Boolean object interface */ #ifndef Py_BOOLOBJECT_H #define Py_BOOLOBJECT_H @@ -7,15 +6,19 @@ extern "C" { #endif -#define Py_False ((PyObject *) &_Py_ZeroStruct) +#define PyBool_Check(x) (Py_TYPE(x) == &PyBool_Type) + +/* Py_False and Py_True are the only two bools in existence. +Don't forget to apply Py_INCREF() when returning either!!! */ + +/* Use these macros */ +#define Py_False ((PyObject *) &_Py_FalseStruct) #define Py_True ((PyObject *) &_Py_TrueStruct) /* Macros for returning Py_True or Py_False, respectively */ #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False -#define PyBool_Check(op) ((op)->ob_type == &PyBool_Type) - #ifdef __cplusplus } #endif From pypy.commits at gmail.com Mon Oct 1 12:22:10 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 01 Oct 2018 09:22:10 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: PyNone_Type and PyNotImplemented_Type are private in py3 Message-ID: <5bb249b2.1c69fb81.a9f3.6e33@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r95175:911406cbc134 Date: 2018-10-01 17:21 +0100 http://bitbucket.org/pypy/pypy/changeset/911406cbc134/ Log: PyNone_Type and PyNotImplemented_Type are private in py3 diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -712,8 +712,8 @@ "PyByteArray_Type": "space.w_bytearray", "PyMemoryView_Type": "space.w_memoryview", "PyBaseObject_Type": "space.w_object", - 'PyNone_Type': 'space.type(space.w_None)', - 'PyNotImplemented_Type': 'space.type(space.w_NotImplemented)', + '_PyNone_Type': 'space.type(space.w_None)', + '_PyNotImplemented_Type': 'space.type(space.w_NotImplemented)', 'PyCell_Type': 'space.gettypeobject(Cell.typedef)', 'PyModule_Type': 'space.gettypeobject(Module.typedef)', 'PyProperty_Type': 'space.gettypeobject(W_Property.typedef)', From pypy.commits at gmail.com Mon Oct 1 15:44:40 2018 From: pypy.commits at gmail.com (rlamy) Date: Mon, 01 Oct 2018 12:44:40 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: kill some PyFile_* functions thyat were removed in py3 Message-ID: <5bb27928.1c69fb81.85664.eb37@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r95176:e6fcda16c6ae Date: 2018-10-01 20:43 +0100 http://bitbucket.org/pypy/pypy/changeset/e6fcda16c6ae/ Log: kill some PyFile_* functions thyat were removed in py3 diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -41,19 +41,6 @@ w_mode = space.newtext(rffi.charp2str(mode)) return space.call_method(space.builtin, 'open', w_filename, w_mode) - at cpython_api([FILEP, CONST_STRING, CONST_STRING, rffi.VOIDP], PyObject) -def PyFile_FromFile(space, fp, name, mode, close): - """Create a new PyFileObject from the already-open standard C file - pointer, fp. The function close will be called when the file should be - closed. Return NULL on failure.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.INT_real], lltype.Void) -def PyFile_SetBufSize(space, w_file, n): - """Available on systems with setvbuf() only. This should only be called - immediately after file object creation.""" - raise NotImplementedError - @cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) def PyFile_WriteString(space, s, w_p): """Write string s to file object p. Return 0 on success or -1 on @@ -75,9 +62,3 @@ w_str = space.repr(w_obj) space.call_method(w_p, "write", w_str) return 0 - - at cpython_api([PyObject], PyObject) -def PyFile_Name(space, w_p): - """Return the name of the file specified by p as a string object.""" - w_name = space.getattr(w_p, space.newtext("name")) - return w_name # borrowed ref, should be a W_StringObject from the file diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -48,17 +48,6 @@ space.call_method(w_file, "close") - def test_file_name(self, space, api): - name = str(udir / "_test_file") - with rffi.scoped_str2charp(name) as filename: - with rffi.scoped_str2charp("wb") as mode: - w_file = api.PyFile_FromString(filename, mode) - assert space.str_w(api.PyFile_Name(w_file)) == name - - @pytest.mark.xfail - def test_file_setbufsize(self, space, api): - api.PyFile_SetBufSize() - def test_file_writestring(self, space, api, capfd): w_stdout = space.sys.get("stdout") with rffi.scoped_str2charp("test\n") as s: From pypy.commits at gmail.com Tue Oct 2 04:05:41 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 02 Oct 2018 01:05:41 -0700 (PDT) Subject: [pypy-commit] cffi default: Close #264 Message-ID: <5bb326d5.1c69fb81.d43e5.f1b8@mx.google.com> Author: Armin Rigo Branch: Changeset: r3166:8ebfcc176be2 Date: 2018-10-02 10:05 +0200 http://bitbucket.org/cffi/cffi/changeset/8ebfcc176be2/ Log: Close #264 diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst --- a/doc/source/embedding.rst +++ b/doc/source/embedding.rst @@ -375,8 +375,10 @@ CPython C extension modules may be compiled without saying that they depend on ``libpythonX.Y.so``. This makes such Python systems unsuitable for embedding if the embedder uses ``dlopen(..., - RTLD_LOCAL)``. You get the error ``undefined symbol: - PyExc_SystemError``. See `issue #264`__. + RTLD_LOCAL)``. You get an ``undefined symbol`` error. See + `issue #264`__. A workaround is to first call + ``dlopen("/path/to/libpythonX.Y.so", RTLD_GLOBAL)``, which will + force ``libpythonX.Y.so`` to be loaded first. .. __: https://bitbucket.org/cffi/cffi/issues/264/ From pypy.commits at gmail.com Tue Oct 2 04:09:14 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 02 Oct 2018 01:09:14 -0700 (PDT) Subject: [pypy-commit] cffi default: Ah, we don't usually need the full path here Message-ID: <5bb327aa.1c69fb81.81bb.69f1@mx.google.com> Author: Armin Rigo Branch: Changeset: r3167:316d5bc342b5 Date: 2018-10-02 10:09 +0200 http://bitbucket.org/cffi/cffi/changeset/316d5bc342b5/ Log: Ah, we don't usually need the full path here diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst --- a/doc/source/embedding.rst +++ b/doc/source/embedding.rst @@ -377,7 +377,7 @@ unsuitable for embedding if the embedder uses ``dlopen(..., RTLD_LOCAL)``. You get an ``undefined symbol`` error. See `issue #264`__. A workaround is to first call - ``dlopen("/path/to/libpythonX.Y.so", RTLD_GLOBAL)``, which will + ``dlopen("libpythonX.Y.so", RTLD_LAZY|RTLD_GLOBAL)``, which will force ``libpythonX.Y.so`` to be loaded first. .. __: https://bitbucket.org/cffi/cffi/issues/264/ From pypy.commits at gmail.com Tue Oct 2 12:37:52 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 02 Oct 2018 09:37:52 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Remove __long__ on py3 Message-ID: <5bb39ee0.1c69fb81.551c7.5efa@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r95177:ade16cd72f6b Date: 2018-10-02 18:29 +0200 http://bitbucket.org/pypy/pypy/changeset/ade16cd72f6b/ Log: Remove __long__ on py3 diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -79,13 +79,6 @@ w_result = self.ctype.cast_to_int(ptr) return w_result - def long(self, space): - w_result = self.int(space) - space = self.space - if space.is_w(space.type(w_result), space.w_int): - w_result = space.newlong(space.int_w(w_result)) - return w_result - def float(self): with self as ptr: w_result = self.ctype.float(ptr) @@ -664,7 +657,6 @@ __repr__ = interp2app(W_CData.repr), __bool__ = interp2app(W_CData.bool), __int__ = interp2app(W_CData.int), - __long__ = interp2app(W_CData.long), __float__ = interp2app(W_CData.float), __complex__ = interp2app(W_CData.complex), __len__ = interp2app(W_CData.len), From pypy.commits at gmail.com Wed Oct 3 05:39:50 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 03 Oct 2018 02:39:50 -0700 (PDT) Subject: [pypy-commit] buildbot default: Stop trying to run the benchmarks on tannit Message-ID: <5bb48e66.1c69fb81.9598f.4f32@mx.google.com> Author: Armin Rigo Branch: Changeset: r1064:2aed4ccd3b31 Date: 2018-10-03 11:39 +0200 http://bitbucket.org/pypy/buildbot/changeset/2aed4ccd3b31/ Log: Stop trying to run the benchmarks on tannit diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -321,8 +321,8 @@ ), Nightly("nightly-1-00", [ - JITBENCH, # on tannit32, uses 1 core (in part exclusively) - JITBENCH64, # on tannit64, uses 1 core (in part exclusively) + #JITBENCH, # on tannit32, uses 1 core (in part exclusively) + #JITBENCH64, # on tannit64, uses 1 core (in part exclusively) #JITBENCH64_NEW, # on speed64, uses 1 core (in part exclusively) ], branch=None, hour=5, minute=0, From pypy.commits at gmail.com Wed Oct 3 13:30:39 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 03 Oct 2018 10:30:39 -0700 (PDT) Subject: [pypy-commit] pypy default: Add C implementations for PyObject_DelItemString, PyMapping_DelItem, PyMapping_DelItemString Message-ID: <5bb4fcbf.1c69fb81.cc97.5b47@mx.google.com> Author: Ronan Lamy Branch: Changeset: r95179:a5735c0b1edd Date: 2018-10-03 18:29 +0100 http://bitbucket.org/pypy/pypy/changeset/a5735c0b1edd/ Log: Add C implementations for PyObject_DelItemString, PyMapping_DelItem, PyMapping_DelItemString diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -603,10 +603,11 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', - 'PyObject_GetBuffer', 'PyBuffer_Release', - 'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', 'PyBuffer_FromObject', - 'PyBuffer_FromReadWriteObject', 'PyBuffer_New', 'PyBuffer_Type', '_Py_get_buffer_type', - '_Py_setfilesystemdefaultencoding', + 'PyObject_DelItemString', 'PyObject_GetBuffer', 'PyBuffer_Release', + + 'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', + 'PyBuffer_FromObject', 'PyBuffer_FromReadWriteObject', 'PyBuffer_New', + 'PyBuffer_Type', '_Py_get_buffer_type', '_Py_setfilesystemdefaultencoding', 'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr', 'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr', @@ -1299,7 +1300,7 @@ # if do tuple_attach of the prebuilt empty tuple, we need to call # _PyPy_Malloc) builder.attach_all(space) - + setup_init_functions(eci, prefix) return modulename.new(ext='') diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -4,6 +4,15 @@ extern "C" { #endif + PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, char *key); + + /* + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ + + /* new buffer API */ #define PyObject_CheckBuffer(obj) \ @@ -28,6 +37,27 @@ /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. */ +/* Mapping protocol:*/ + + /* implemented as a macro: + + int PyMapping_DelItemString(PyObject *o, char *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItemString(O,K) PyObject_DelItemString((O),(K)) + + /* implemented as a macro: + + int PyMapping_DelItem(PyObject *o, PyObject *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItem(O,K) PyObject_DelItem((O),(K)) #ifdef __cplusplus } diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -23,6 +23,23 @@ /* Operations on any object */ int +PyObject_DelItemString(PyObject *o, char *key) +{ + PyObject *okey; + int ret; + + if (o == NULL || key == NULL) { + null_error(); + return -1; + } + okey = PyString_FromString(key); + if (okey == NULL) + return -1; + ret = PyObject_DelItem(o, okey); + Py_DECREF(okey); + return ret; +} +int PyObject_CheckReadBuffer(PyObject *obj) { PyBufferProcs *pb = obj->ob_type->tp_as_buffer; @@ -101,6 +118,20 @@ return 0; } +/* Buffer C-API for Python 3.0 */ + +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) { @@ -116,6 +147,7 @@ return (void*)pointer; } + void _Py_add_one_to_index_F(int nd, Py_ssize_t *index, const Py_ssize_t *shape) { @@ -258,19 +290,6 @@ -/* Buffer C-API for Python 3.0 */ - -int -PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) -{ - if (!PyObject_CheckBuffer(obj)) { - PyErr_Format(PyExc_TypeError, - "'%100s' does not have the buffer interface", - Py_TYPE(obj)->tp_name); - return -1; - } - return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); -} void PyBuffer_Release(Py_buffer *view) @@ -428,6 +447,7 @@ return retval; } + static PyObject * objargs_mktuple(va_list va) { diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1233,18 +1233,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) -def PyMapping_DelItemString(space, o, key): - """Remove the mapping for object key from the object o. Return -1 on - failure. This is equivalent to the Python statement del o[key].""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PyMapping_DelItem(space, o, key): - """Remove the mapping for object key from the object o. Return -1 on - failure. This is equivalent to the Python statement del o[key].""" - raise NotImplementedError - @cpython_api([lltype.Signed, FILE, rffi.INT_real], lltype.Void) def PyMarshal_WriteLongToFile(space, value, file, version): """Marshal a long integer, value, to file. This will only write From pypy.commits at gmail.com Sun Oct 7 06:23:24 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Oct 2018 03:23:24 -0700 (PDT) Subject: [pypy-commit] pypy default: add, test LoadLibraryW, LoadLibraryEx{A, W} Message-ID: <5bb9de9c.1c69fb81.52037.8614@mx.google.com> Author: Matti Picus Branch: Changeset: r95180:80ba133be397 Date: 2018-10-07 12:10 +0300 http://bitbucket.org/pypy/pypy/changeset/80ba133be397/ Log: add, test LoadLibraryW, LoadLibraryEx{A, W} diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -113,6 +113,7 @@ MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR + LOAD_WITH_ALTERED_SEARCH_PATH """ from rpython.translator.platform import host_factory static_platform = host_factory() @@ -195,6 +196,22 @@ GetModuleHandle = winexternal('GetModuleHandleA', [rffi.CCHARP], HMODULE) LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) + def wrap_loadlibraryex(func): + def loadlibrary(name, handle=None, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + # Requires a full path name with '/' -> '\\' + return func(name, handle, flags) + return loadlibrary + + _LoadLibraryExA = winexternal('LoadLibraryExA', + [rffi.CCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExA = wrap_loadlibraryex(_LoadLibraryExA) + LoadLibraryW = winexternal('LoadLibraryW', [rffi.CWCHARP], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + _LoadLibraryExW = winexternal('LoadLibraryExW', + [rffi.CWCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExW = wrap_loadlibraryex(_LoadLibraryExW) GetProcAddress = winexternal('GetProcAddress', [HMODULE, rffi.CCHARP], rffi.VOIDP) diff --git a/rpython/rlib/test/loadtest/loadtest0.dll b/rpython/rlib/test/loadtest/loadtest0.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9bdcc33a1902f8e989d349c49c2cc08e633aa32b GIT binary patch [cut] diff --git a/rpython/rlib/test/loadtest/loadtest1.dll b/rpython/rlib/test/loadtest/loadtest1.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cb83854875c876717371bdf90488eed9c6571f03 GIT binary patch [cut] diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -6,6 +6,44 @@ from rpython.rlib import rwin32 from rpython.tool.udir import udir +loadtest_dir = os.path.dirname(__file__) + '/loadtest' +test1 = os.path.abspath(loadtest_dir + '/loadtest1.dll') +test0 = os.path.abspath(loadtest_dir + '/loadtest0.dll') + +if not os.path.exists(test1) or not os.path.exists(test0): + # This is how the files, which are checked into the repo, were created + from rpython.translator.tool.cbuild import ExternalCompilationInfo + from rpython.translator.platform import platform + from rpython.translator import cdir + if not os.path.exists(loadtest_dir): + os.mkdir(loadtest_dir) + c_file = udir.ensure("test_rwin32", dir=1).join("test0.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + RPY_EXPORTED + int internal_sum(int a, int b) { + return a + b; + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir]) + lib_name = str(platform.compile([c_file], eci, test0[:-4], + standalone=False)) + assert os.path.abspath(lib_name) == os.path.abspath(test0) + + c_file = udir.ensure("test_rwin32", dir=1).join("test1.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + int internal_sum(int a, int b); + RPY_EXPORTED + int sum(int a, int b) { + return internal_sum(a, b); + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir], + libraries=[loadtest_dir + '/loadtest0']) + lib_name = str(platform.compile([c_file], eci, test1[:-4], + standalone=False, )) + assert os.path.abspath(lib_name) == os.path.abspath(test1) def test_get_osfhandle(): fid = open(str(udir.join('validate_test.txt')), 'w') @@ -28,13 +66,13 @@ "import time;" "time.sleep(10)", ], - ) + ) print proc.pid handle = rwin32.OpenProcess(rwin32.PROCESS_ALL_ACCESS, False, proc.pid) assert rwin32.TerminateProcess(handle, signal.SIGTERM) == 1 rwin32.CloseHandle(handle) assert proc.wait() == signal.SIGTERM - + @py.test.mark.dont_track_allocations('putenv intentionally keeps strings alive') def test_wenviron(): name, value = u'PYPY_TEST_日本', u'foobar日本' @@ -55,3 +93,48 @@ msg = rwin32.FormatErrorW(34) assert type(msg) is unicode assert u'%2' in msg + +def test_loadlibraryA(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibrary(test0) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibrary(test1) + assert not hdll + + assert os.path.exists(test1) + + hdll = rwin32.LoadLibraryExA(test1) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + +def test_loadlibraryW(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibraryW(unicode(test0)) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibraryW(unicode(test1)) + assert not hdll + + assert os.path.exists(unicode(test1)) + + hdll = rwin32.LoadLibraryExW(unicode(test1)) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + +def test_loadlibrary_unicode(): + import shutil + test0u = unicode(udir.join(u'load\u03betest.dll')) + shutil.copyfile(test0, test0u) + hdll = rwin32.LoadLibraryW(test0u) + assert hdll From pypy.commits at gmail.com Sun Oct 7 06:23:26 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Oct 2018 03:23:26 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: add failing test Message-ID: <5bb9de9e.1c69fb81.e84d0.3c4e@mx.google.com> Author: Matti Picus Branch: cffi_dlopen_unicode Changeset: r95181:ec14b51b9207 Date: 2018-10-07 13:03 +0300 http://bitbucket.org/pypy/pypy/changeset/ec14b51b9207/ Log: add failing test diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,4 +1,5 @@ import py +import shutil from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache @@ -40,6 +41,9 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) + outputfileUname = unicode(udir.join(u'load\u03betest.dll')) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -108,6 +112,13 @@ assert lib.add42(-10) == 32 assert type(lib.add42) is _cffi_backend.FFI.CData + def test_dlopen_unicode(self): + import _cffi_backend + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 + def test_dlclose(self): import _cffi_backend self.fix_path() From pypy.commits at gmail.com Sun Oct 7 07:21:02 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Oct 2018 04:21:02 -0700 (PDT) Subject: [pypy-commit] pypy default: Use LoadLibraryEx in loading extension modules, like CPython's _PyImport_FindSharedFuncptr Message-ID: <5bb9ec1e.1c69fb81.f36f7.1478@mx.google.com> Author: Matti Picus Branch: Changeset: r95182:8894e5f1d5c0 Date: 2018-10-07 14:20 +0300 http://bitbucket.org/pypy/pypy/changeset/8894e5f1d5c0/ Log: Use LoadLibraryEx in loading extension modules, like CPython's _PyImport_FindSharedFuncptr diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -92,8 +92,11 @@ if sys.platform == 'win32': dash = '_' + WIN32 = True else: dash = '' + WIN32 = False + def fclose(fp): try: @@ -1656,7 +1659,11 @@ try: ll_libname = rffi.str2charp(path) try: - dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) + if WIN32: + # Allow other DLLs in the same directory with "path" + dll = rdynload.dlopenex(ll_libname) + else: + dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -233,6 +233,15 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenex(name): + res = rwin32.LoadLibraryExA(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: From pypy.commits at gmail.com Sun Oct 7 09:30:56 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Oct 2018 06:30:56 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: allow unicode filename in W_DlOpenLibObject Message-ID: <5bba0a90.1c69fb81.51cfc.b9ed@mx.google.com> Author: Matti Picus Branch: cffi_dlopen_unicode Changeset: r95183:9093671404b4 Date: 2018-10-07 16:30 +0300 http://bitbucket.org/pypy/pypy/changeset/9093671404b4/ Log: allow unicode filename in W_DlOpenLibObject diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,3 +1,4 @@ +import sys from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError @@ -12,20 +13,36 @@ from pypy.module._cffi_backend.lib_obj import W_LibObject from pypy.module._cffi_backend import cffi_opcode, cffi1_module +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False class W_DlOpenLibObject(W_LibObject): - def __init__(self, ffi, filename, flags): - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" + def __init__(self, ffi, w_filename, flags): + space = ffi.space + _scoped = rffi.scoped_str2charp + _dlopen = dlopen + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + _scoped = rffi.scoped_unicode2wcharp + _dlopen = dlopenU + fname = space.unicode_w(w_filename) + elif space.is_none(w_filename): + fname = None + else: + fname = space.text_w(w_filename) + with _scoped(fname) as ll_libname: + if fname is None: + fname = "" try: - handle = dlopen(ll_libname, flags) + handle = _dlopen(ll_libname, flags) except DLOpenError as e: - raise wrap_dlopenerror(ffi.space, e, filename) - W_LibObject.__init__(self, ffi, filename) + raise wrap_dlopenerror(space, e, fname) + W_LibObject.__init__(self, ffi, fname) self.libhandle = handle - self.register_finalizer(ffi.space) + self.register_finalizer(space) def _finalize_(self): h = self.libhandle diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -572,8 +572,8 @@ return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA) - @unwrap_spec(filename="fsencode_or_none", flags=int) - def descr_dlopen(self, filename, flags=0): + @unwrap_spec(flags=int) + def descr_dlopen(self, w_filename, flags=0): """\ Load and return a dynamic library identified by 'name'. The standard C library can be loaded by passing None. @@ -584,7 +584,7 @@ first access.""" # from pypy.module._cffi_backend import cdlopen - return cdlopen.W_DlOpenLibObject(self, filename, flags) + return cdlopen.W_DlOpenLibObject(self, w_filename, flags) def descr_dlclose(self, w_lib): diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -233,6 +233,16 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenU(name, mode=-1): + # mode is unused on windows, but a consistant signature + res = rwin32.LoadLibraryW(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: From pypy.commits at gmail.com Sun Oct 7 10:39:40 2018 From: pypy.commits at gmail.com (arigo) Date: Sun, 07 Oct 2018 07:39:40 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: #2900 fix in the _ssl module with poll() Message-ID: <5bba1aac.1c69fb81.bd692.1c35@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r95184:000e19254270 Date: 2018-10-07 16:39 +0200 http://bitbucket.org/pypy/pypy/changeset/000e19254270/ Log: #2900 fix in the _ssl module with poll() diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -186,7 +186,7 @@ # Prefer poll, if available, since you can poll() any fd # which can't be done with select(). if HAVE_POLL: - p.register(sock.fileno(), POLLOUT | POLLIN) + p.register(sock.fileno(), POLLOUT if writing else POLLIN) rc = len(p.poll(timeout * 1000.0)) else: From pypy.commits at gmail.com Sun Oct 7 13:33:34 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Oct 2018 10:33:34 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: rework to fix win32 translation Message-ID: <5bba436e.1c69fb81.fc03e.de3e@mx.google.com> Author: Matti Picus Branch: cffi_dlopen_unicode Changeset: r95185:7013b39caaa8 Date: 2018-10-07 20:33 +0300 http://bitbucket.org/pypy/pypy/changeset/7013b39caaa8/ Log: rework to fix win32 translation diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -23,23 +23,26 @@ def __init__(self, ffi, w_filename, flags): space = ffi.space - _scoped = rffi.scoped_str2charp - _dlopen = dlopen if WIN32 and space.isinstance_w(w_filename, space.w_unicode): - _scoped = rffi.scoped_unicode2wcharp - _dlopen = dlopenU fname = space.unicode_w(w_filename) - elif space.is_none(w_filename): - fname = None + with rffi.scoped_unicode2wcharp(fname) as ll_libname: + fname = fname.encode('utf-8') + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) else: - fname = space.text_w(w_filename) - with _scoped(fname) as ll_libname: - if fname is None: - fname = "" - try: - handle = _dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, fname) + if space.is_none(w_filename): + fname = None + else: + fname = space.text_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) W_LibObject.__init__(self, ffi, fname) self.libhandle = handle self.register_finalizer(space) From pypy.commits at gmail.com Sun Oct 7 14:19:25 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Oct 2018 11:19:25 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: fix for non-win32 Message-ID: <5bba4e2d.1c69fb81.b02bf.a0b7@mx.google.com> Author: Matti Picus Branch: cffi_dlopen_unicode Changeset: r95186:da2aa8584393 Date: 2018-10-07 20:56 +0300 http://bitbucket.org/pypy/pypy/changeset/da2aa8584393/ Log: fix for non-win32 diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -34,6 +34,8 @@ else: if space.is_none(w_filename): fname = None + elif space.isinstance_w(w_filename, space.w_unicode): + fname = space.unicode_w(w_filename).encode('utf-8') else: fname = space.text_w(w_filename) with rffi.scoped_str2charp(fname) as ll_libname: From pypy.commits at gmail.com Mon Oct 8 02:41:43 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 07 Oct 2018 23:41:43 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: copy unicode handling to _cffi_backend.libraryobj.W_Library Message-ID: <5bbafc27.1c69fb81.95ede.a32a@mx.google.com> Author: Matti Picus Branch: cffi_dlopen_unicode Changeset: r95187:295154c77400 Date: 2018-10-08 09:36 +0300 http://bitbucket.org/pypy/pypy/changeset/295154c77400/ Log: copy unicode handling to _cffi_backend.libraryobj.W_Library diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -1,4 +1,5 @@ from __future__ import with_statement +import sys from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import oefmt @@ -12,20 +13,42 @@ from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + class W_Library(W_Root): _immutable_ = True - def __init__(self, space, filename, flags): + def __init__(self, space, w_filename, flags): self.space = space - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - self.handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, filename) - self.name = filename + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(fname) as ll_libname: + fname = fname.encode('utf-8') + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + elif space.isinstance_w(w_filename, space.w_unicode): + fname = space.unicode_w(w_filename).encode('utf-8') + else: + fname = space.text_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + self.handle = handle + self.name = fname self.register_finalizer(space) def _finalize_(self): @@ -104,7 +127,7 @@ W_Library.typedef.acceptable_as_base_class = False - at unwrap_spec(filename="fsencode_or_none", flags=int) -def load_library(space, filename, flags=0): - lib = W_Library(space, filename, flags) + at unwrap_spec(flags=int) +def load_library(space, w_filename, flags=0): + lib = W_Library(space, w_filename, flags) return lib From pypy.commits at gmail.com Mon Oct 8 05:58:01 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 08 Oct 2018 02:58:01 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: update other users of W_Library to W_Root interface Message-ID: <5bbb2a29.1c69fb81.cc67d.9386@mx.google.com> Author: Matti Picus Branch: cffi_dlopen_unicode Changeset: r95188:0e3e1d314c39 Date: 2018-10-08 12:55 +0300 http://bitbucket.org/pypy/pypy/changeset/0e3e1d314c39/ Log: update other users of W_Library to W_Root interface diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -308,10 +308,10 @@ dldflags = rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY if os.environ.get('CPPYY_BACKEND_LIBRARY'): libname = os.environ['CPPYY_BACKEND_LIBRARY'] - state.backend = W_Library(space, libname, dldflags) + state.backend = W_Library(space, space.newtext(libname), dldflags) else: # try usual lookups - state.backend = W_Library(space, backend_library, dldflags) + state.backend = W_Library(space, space.newtext(backend_library), dldflags) if state.backend: # fix constants From pypy.commits at gmail.com Mon Oct 8 10:35:07 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 08 Oct 2018 07:35:07 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: skip unicode file test on non-windows Message-ID: <5bbb6b1b.1c69fb81.5665a.abd8@mx.google.com> Author: Matti Picus Branch: cffi_dlopen_unicode Changeset: r95189:943912f5c5dd Date: 2018-10-08 17:34 +0300 http://bitbucket.org/pypy/pypy/changeset/943912f5c5dd/ Log: skip unicode file test on non-windows diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,9 +1,13 @@ import py -import shutil +import sys, shutil from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache +if sys.platform == 'win32': + WIN32 = True +else: + WIN32 = False class AppTestRecompilerPython: spaceconfig = dict(usemodules=['_cffi_backend']) @@ -41,9 +45,11 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) - outputfileUname = unicode(udir.join(u'load\u03betest.dll')) - shutil.copyfile(outputfilename, outputfileUname) - cls.w_extmodU = space.wrap(outputfileUname) + if WIN32: + # non-windows need utf8 locales to deal with unicode filenames + outputfileUname = unicode(udir.join(u'load\u03betest.dll')) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -113,11 +119,12 @@ assert type(lib.add42) is _cffi_backend.FFI.CData def test_dlopen_unicode(self): - import _cffi_backend - self.fix_path() - from re_python_pysrc import ffi - lib = ffi.dlopen(self.extmodU) - assert lib.add42(-10) == 32 + if getattr(self, extmodU, None): + import _cffi_backend + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 def test_dlclose(self): import _cffi_backend From pypy.commits at gmail.com Mon Oct 8 12:14:58 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Oct 2018 09:14:58 -0700 (PDT) Subject: [pypy-commit] pypy default: Remove from stubs.py the few functions that are implemented. Message-ID: <5bbb8282.1c69fb81.66935.2f4b@mx.google.com> Author: Armin Rigo Branch: Changeset: r95190:91a8a20e0809 Date: 2018-10-08 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/91a8a20e0809/ Log: Remove from stubs.py the few functions that are implemented. Checked in the script I used to find them. diff --git a/pypy/module/cpyext/stubs-find-implemented.py b/pypy/module/cpyext/stubs-find-implemented.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/stubs-find-implemented.py @@ -0,0 +1,21 @@ +import re +import os + + +for line in open('stubs.py'): + if not line.strip(): + continue + if line.startswith(' '): + continue + if line.startswith('#'): + continue + if line.startswith('@cpython_api'): + continue + if line.endswith(' = rffi.VOIDP\n'): + continue + + #print line.rstrip() + m = re.match(r"def ([\w\d_]+)[(]", line) + assert m, line + funcname = m.group(1) + os.system('grep -w %s [a-r]*.py s[a-s]*.py str*.py stubsa*.py sy*.py [t-z]*.py' % funcname) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1,26 +1,15 @@ -from pypy.module.cpyext.api import ( - cpython_api, PyObject, PyObjectP, CANNOT_FAIL - ) -from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex -from rpython.rtyper.lltypesystem import rffi, lltype +#----this file is not imported, only here for reference---- -# we don't really care -PyTypeObjectPtr = rffi.VOIDP -Py_ssize_t = rffi.SSIZE_T -PyMethodDef = rffi.VOIDP -PyGetSetDef = rffi.VOIDP -PyMemberDef = rffi.VOIDP -va_list = rffi.VOIDP +#from pypy.module.cpyext.api import ( +# cpython_api, PyObject, PyObjectP, CANNOT_FAIL +# ) +#from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex +#from rpython.rtyper.lltypesystem import rffi, lltype + wrapperbase = rffi.VOIDP FILE = rffi.VOIDP PyFileObject = rffi.VOIDP -PyCodeObject = rffi.VOIDP -PyFrameObject = rffi.VOIDP _inittab = rffi.VOIDP -PyThreadState = rffi.VOIDP -PyInterpreterState = rffi.VOIDP -Py_UNICODE = lltype.UniChar -PyCompilerFlags = rffi.VOIDP _node = rffi.VOIDP Py_tracefunc = rffi.VOIDP @@ -254,39 +243,6 @@ instead.""" raise NotImplementedError - at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP) -def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): - """Convert a double val to a string using supplied - format_code, precision, and flags. - - format_code must be one of 'e', 'E', 'f', 'F', - 'g', 'G' or 'r'. For 'r', the supplied precision - must be 0 and is ignored. The 'r' format code specifies the - standard repr() format. - - flags can be zero or more of the values Py_DTSF_SIGN, - Py_DTSF_ADD_DOT_0, or Py_DTSF_ALT, or-ed together: - - Py_DTSF_SIGN means to always precede the returned string with a sign - character, even if val is non-negative. - - Py_DTSF_ADD_DOT_0 means to ensure that the returned string will not look - like an integer. - - Py_DTSF_ALT means to apply "alternate" formatting rules. See the - documentation for the PyOS_snprintf() '#' specifier for - details. - - If ptype is non-NULL, then the value it points to will be set to one of - Py_DTST_FINITE, Py_DTST_INFINITE, or Py_DTST_NAN, signifying that - val is a finite number, an infinite number, or not a number, respectively. - - The return value is a pointer to buffer with the converted string or - NULL if the conversion failed. The caller is responsible for freeing the - returned string by calling PyMem_Free(). - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP], rffi.DOUBLE, error=CANNOT_FAIL) def PyOS_ascii_atof(space, nptr): """Convert a string to a double in a locale-independent way. @@ -310,24 +266,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_Check(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType or a subtype of - PyDateTime_TZInfoType. ob must not be NULL. - """ - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_CheckExact(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType. ob must not be - NULL. - """ - raise NotImplementedError - - at cpython_api([PyTypeObjectPtr, PyGetSetDef], PyObject) -def PyDescr_NewGetSet(space, type, getset): - raise NotImplementedError - @cpython_api([PyTypeObjectPtr, PyMemberDef], PyObject) def PyDescr_NewMember(space, type, meth): raise NotImplementedError @@ -1206,14 +1144,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.ULONGLONG, error=-1) -def PyInt_AsUnsignedLongLongMask(space, io): - """Will first attempt to cast the object to a PyIntObject or - PyLongObject, if it is not already one, and then return its value as - unsigned long long, without checking for overflow. - """ - raise NotImplementedError - @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyInt_ClearFreeList(space): """Clear the integer free list. Return the number of items that could not From pypy.commits at gmail.com Mon Oct 8 14:05:28 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Oct 2018 11:05:28 -0700 (PDT) Subject: [pypy-commit] pypy default: implement PyEval_GetFrame() Message-ID: <5bbb9c68.1c69fb81.b02bf.ae64@mx.google.com> Author: Armin Rigo Branch: Changeset: r95191:1d9271dd9a3d Date: 2018-10-08 19:58 +0200 http://bitbucket.org/pypy/pypy/changeset/1d9271dd9a3d/ Log: implement PyEval_GetFrame() diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject +from pypy.module.cpyext.frameobject import PyFrameObject from pypy.module.__builtin__ import compiling PyCompilerFlags = cpython_struct( @@ -58,6 +59,11 @@ return None return caller.get_w_globals() # borrowed ref + at cpython_api([], PyFrameObject, error=CANNOT_FAIL, result_borrowed=True) +def PyEval_GetFrame(space): + caller = space.getexecutioncontext().gettopframe_nohidden() + return caller # borrowed ref, may be null + @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): """This is a simplified interface to PyEval_EvalCodeEx(), with just diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -420,3 +420,15 @@ except StopIteration: pass assert out == [0, 1, 2, 3, 4] + + def test_getframe(self): + import sys + module = self.import_extension('foo', [ + ("getframe1", "METH_NOARGS", + """ + PyFrameObject *x = PyEval_GetFrame(); + Py_INCREF(x); + return (PyObject *)x; + """),], prologue="#include \n") + res = module.getframe1() + assert res is sys._getframe(0) From pypy.commits at gmail.com Mon Oct 8 14:06:25 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Oct 2018 11:06:25 -0700 (PDT) Subject: [pypy-commit] pypy default: Remove PyEval_GetFrame() from stubs Message-ID: <5bbb9ca1.1c69fb81.68c7.611a@mx.google.com> Author: Armin Rigo Branch: Changeset: r95192:0f86b89b1d81 Date: 2018-10-08 20:05 +0200 http://bitbucket.org/pypy/pypy/changeset/0f86b89b1d81/ Log: Remove PyEval_GetFrame() from stubs diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1263,13 +1263,6 @@ for PyObject_Str().""" raise NotImplementedError - at cpython_api([], PyFrameObject) -def PyEval_GetFrame(space): - """Return the current thread state's frame, which is NULL if no frame is - currently executing.""" - borrow_from() - raise NotImplementedError - @cpython_api([PyFrameObject], rffi.INT_real, error=CANNOT_FAIL) def PyFrame_GetLineNumber(space, frame): """Return the line number that frame is currently executing.""" From pypy.commits at gmail.com Mon Oct 8 14:33:40 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Oct 2018 11:33:40 -0700 (PDT) Subject: [pypy-commit] pypy default: Not importable anyway, so remove the remaining dummy types Message-ID: <5bbba304.1c69fb81.c8460.406f@mx.google.com> Author: Armin Rigo Branch: Changeset: r95193:1a6879342343 Date: 2018-10-08 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/1a6879342343/ Log: Not importable anyway, so remove the remaining dummy types diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -6,12 +6,6 @@ #from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex #from rpython.rtyper.lltypesystem import rffi, lltype -wrapperbase = rffi.VOIDP -FILE = rffi.VOIDP -PyFileObject = rffi.VOIDP -_inittab = rffi.VOIDP -_node = rffi.VOIDP -Py_tracefunc = rffi.VOIDP @cpython_api([rffi.CCHARP], Py_ssize_t, error=CANNOT_FAIL) def PyBuffer_SizeFromFormat(space, format): From pypy.commits at gmail.com Mon Oct 8 14:33:43 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 08 Oct 2018 11:33:43 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5bbba307.1c69fb81.5194e.b0bc@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r95194:248ed5fa4ded Date: 2018-10-08 20:30 +0200 http://bitbucket.org/pypy/pypy/changeset/248ed5fa4ded/ Log: hg merge default diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -761,6 +761,26 @@ assert b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1") == \ u"\u3042[<92><117><51>]xxx" + def test_unicode_internal_error_handler_infinite_loop(self): + import codecs + class MyException(Exception): + pass + seen = [0] + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + seen[0] += 1 + if seen[0] == 20: # stop the 20th time this is called + raise MyException + return (u"\x01", 4) # 4 < len(input), so will try and fail again + codecs.register_error("test.inf", handler_unicodeinternal) + try: + b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.inf") + except MyException: + pass + else: + raise AssertionError("should have gone into infinite loop") + def test_encode_error_bad_handler(self): import codecs codecs.register_error("test.bad_handler", lambda e: (repl, 1)) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -93,8 +93,11 @@ if sys.platform == 'win32': dash = '_' + WIN32 = True else: dash = '' + WIN32 = False + def fclose(fp): try: @@ -610,7 +613,7 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', - 'PyObject_GetBuffer', 'PyBuffer_Release', + 'PyObject_DelItemString', 'PyObject_GetBuffer', 'PyBuffer_Release', '_Py_setfilesystemdefaultencoding', 'PyCapsule_New', 'PyCapsule_IsValid', 'PyCapsule_GetPointer', @@ -1664,7 +1667,11 @@ try: ll_libname = rffi.str2charp(path) try: - dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) + if WIN32: + # Allow other DLLs in the same directory with "path" + dll = rdynload.dlopenex(ll_libname) + else: + dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject +from pypy.module.cpyext.frameobject import PyFrameObject from pypy.module.__builtin__ import compiling PyCompilerFlags = cpython_struct( @@ -58,6 +59,11 @@ return None return caller.get_w_globals() # borrowed ref + at cpython_api([], PyFrameObject, error=CANNOT_FAIL, result_borrowed=True) +def PyEval_GetFrame(space): + caller = space.getexecutioncontext().gettopframe_nohidden() + return caller # borrowed ref, may be null + @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): """This is a simplified interface to PyEval_EvalCodeEx(), with just diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -4,6 +4,15 @@ extern "C" { #endif + PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, char *key); + + /* + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ + + /* new buffer API */ #define PyObject_CheckBuffer(obj) \ @@ -27,6 +36,27 @@ /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. */ +/* Mapping protocol:*/ + + /* implemented as a macro: + + int PyMapping_DelItemString(PyObject *o, char *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItemString(O,K) PyObject_DelItemString((O),(K)) + + /* implemented as a macro: + + int PyMapping_DelItem(PyObject *o, PyObject *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItem(O,K) PyObject_DelItem((O),(K)) #ifdef __cplusplus } diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -23,6 +23,23 @@ /* Operations on any object */ int +PyObject_DelItemString(PyObject *o, char *key) +{ + PyObject *okey; + int ret; + + if (o == NULL || key == NULL) { + null_error(); + return -1; + } + okey = PyUnicode_FromString(key); + if (okey == NULL) + return -1; + ret = PyObject_DelItem(o, okey); + Py_DECREF(okey); + return ret; +} +int PyObject_CheckReadBuffer(PyObject *obj) { PyBufferProcs *pb = obj->ob_type->tp_as_buffer; @@ -96,6 +113,20 @@ return 0; } +/* Buffer C-API for Python 3.0 */ + +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) { @@ -111,6 +142,7 @@ return (void*)pointer; } + void _Py_add_one_to_index_F(int nd, Py_ssize_t *index, const Py_ssize_t *shape) { @@ -253,19 +285,6 @@ -/* Buffer C-API for Python 3.0 */ - -int -PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) -{ - if (!PyObject_CheckBuffer(obj)) { - PyErr_Format(PyExc_TypeError, - "'%100s' does not have the buffer interface", - Py_TYPE(obj)->tp_name); - return -1; - } - return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); -} void PyBuffer_Release(Py_buffer *view) @@ -427,6 +446,7 @@ return retval; } + static PyObject * objargs_mktuple(va_list va) { diff --git a/pypy/module/cpyext/stubs-find-implemented.py b/pypy/module/cpyext/stubs-find-implemented.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/stubs-find-implemented.py @@ -0,0 +1,21 @@ +import re +import os + + +for line in open('stubs.py'): + if not line.strip(): + continue + if line.startswith(' '): + continue + if line.startswith('#'): + continue + if line.startswith('@cpython_api'): + continue + if line.endswith(' = rffi.VOIDP\n'): + continue + + #print line.rstrip() + m = re.match(r"def ([\w\d_]+)[(]", line) + assert m, line + funcname = m.group(1) + os.system('grep -w %s [a-r]*.py s[a-s]*.py str*.py stubsa*.py sy*.py [t-z]*.py' % funcname) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1,29 +1,10 @@ -from pypy.module.cpyext.api import ( - cpython_api, PyObject, PyObjectP, CANNOT_FAIL - ) -from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex -from rpython.rtyper.lltypesystem import rffi, lltype +#----this file is not imported, only here for reference---- -CWCHARPP = lltype.Ptr(lltype.Array(rffi.CWCHARP, hints={'nolength': True})) - -# we don't really care -PyTypeObjectPtr = rffi.VOIDP -Py_ssize_t = rffi.SSIZE_T -PyModuleDef = rffi.VOIDP -PyMethodDef = rffi.VOIDP -PyGetSetDef = rffi.VOIDP -PyMemberDef = rffi.VOIDP -va_list = rffi.VOIDP -wrapperbase = rffi.VOIDP -FILE = rffi.VOIDP -PyFrameObject = rffi.VOIDP -_inittab = rffi.VOIDP -PyThreadState = rffi.VOIDP -PyInterpreterState = rffi.VOIDP -Py_UNICODE = lltype.UniChar -PyCompilerFlags = rffi.VOIDP -struct_node = rffi.VOIDP -Py_tracefunc = rffi.VOIDP +#from pypy.module.cpyext.api import ( +# cpython_api, PyObject, PyObjectP, CANNOT_FAIL +# ) +#from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex +#from rpython.rtyper.lltypesystem import rffi, lltype @cpython_api([rffi.CCHARP], Py_ssize_t, error=-1) @@ -228,39 +209,6 @@ this method returns zero and sets errno to EDOM.""" raise NotImplementedError - at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP) -def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): - """Convert a double val to a string using supplied - format_code, precision, and flags. - - format_code must be one of 'e', 'E', 'f', 'F', - 'g', 'G' or 'r'. For 'r', the supplied precision - must be 0 and is ignored. The 'r' format code specifies the - standard repr() format. - - flags can be zero or more of the values Py_DTSF_SIGN, - Py_DTSF_ADD_DOT_0, or Py_DTSF_ALT, or-ed together: - - Py_DTSF_SIGN means to always precede the returned string with a sign - character, even if val is non-negative. - - Py_DTSF_ADD_DOT_0 means to ensure that the returned string will not look - like an integer. - - Py_DTSF_ALT means to apply "alternate" formatting rules. See the - documentation for the PyOS_snprintf() '#' specifier for - details. - - If ptype is non-NULL, then the value it points to will be set to one of - Py_DTST_FINITE, Py_DTST_INFINITE, or Py_DTST_NAN, signifying that - val is a finite number, an infinite number, or not a number, respectively. - - The return value is a pointer to buffer with the converted string or - NULL if the conversion failed. The caller is responsible for freeing the - returned string by calling PyMem_Free(). - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) def PyOS_stricmp(space, s1, s2): """Case insensitive comparison of strings. The function works almost @@ -275,24 +223,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_Check(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType or a subtype of - PyDateTime_TZInfoType. ob must not be NULL. - """ - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_CheckExact(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType. ob must not be - NULL. - """ - raise NotImplementedError - - at cpython_api([PyTypeObjectPtr, PyGetSetDef], PyObject) -def PyDescr_NewGetSet(space, type, getset): - raise NotImplementedError - @cpython_api([PyTypeObjectPtr, PyMemberDef], PyObject) def PyDescr_NewMember(space, type, meth): raise NotImplementedError @@ -483,31 +413,6 @@ 0 on success, -1 on failure.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=-1) -def Py_ReprEnter(space, object): - """Called at the beginning of the tp_repr implementation to - detect cycles. - - If the object has already been processed, the function returns a - positive integer. In that case the tp_repr implementation - should return a string object indicating a cycle. As examples, - dict objects return {...} and list objects - return [...]. - - The function will return a negative integer if the recursion limit - is reached. In that case the tp_repr implementation should - typically return NULL. - - Otherwise, the function returns zero and the tp_repr - implementation can continue normally.""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) -def Py_ReprLeave(space, object): - """Ends a Py_ReprEnter(). Must be called once for each - invocation of Py_ReprEnter() that returns zero.""" - raise NotImplementedError - @cpython_api([rffi.INT_real, rffi.CCHARP, rffi.CCHARP, rffi.INT_real, rffi.CCHARP, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFd(space, fd, name, mode, buffering, encoding, errors, newline, closefd): """Create a Python file object from the file descriptor of an already @@ -1295,39 +1200,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.VOIDP) -def PyModule_GetState(space, module): - """Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or NULL. See - PyModuleDef.m_size.""" - raise NotImplementedError - - at cpython_api([PyObject], PyModuleDef) -def PyModule_GetDef(space, module): - """Return a pointer to the PyModuleDef struct from which the module was - created, or NULL if the module wasn't created with - PyModule_Create().""" - raise NotImplementedError - - - at cpython_api([PyModuleDef], PyObject) -def PyModule_Create(space, module): - """Create a new module object, given the definition in module. This behaves - like PyModule_Create2() with module_api_version set to - PYTHON_API_VERSION.""" - raise NotImplementedError - - - at cpython_api([PyModuleDef, rffi.INT_real], PyObject) -def PyModule_Create2(space, module, module_api_version): - """Create a new module object, given the definition in module, assuming the - API version module_api_version. If that version does not match the version - of the running interpreter, a RuntimeWarning is emitted. - - Most uses of this function should be using PyModule_Create() - instead; only use this if you are sure you need it.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.INT_real], PyObject) def PyNumber_ToBase(space, n, base): """Returns the integer n converted to base base as a string. The base @@ -1337,23 +1209,6 @@ PyNumber_Index() first.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyObject_Bytes(space, o): - """ - Compute a bytes representation of object o. NULL is returned on - failure and a bytes object on success. This is equivalent to the Python - expression bytes(o), when o is not an integer. Unlike bytes(o), - a TypeError is raised when o is an integer instead of a zero-initialized - bytes object.""" - raise NotImplementedError - - at cpython_api([], PyFrameObject) -def PyEval_GetFrame(space): - """Return the current thread state's frame, which is NULL if no frame is - currently executing.""" - raise NotImplementedError - borrow_from() - @cpython_api([PyFrameObject], rffi.INT_real, error=-1) def PyFrame_GetLineNumber(space, frame): """Return the line number that frame is currently executing.""" diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -510,3 +510,15 @@ assert run_async(list1()) == ([], [0, 1, 2]) assert run_async(list2()) == ([], [0, 1, 2]) """ + + def test_getframe(self): + import sys + module = self.import_extension('foo', [ + ("getframe1", "METH_NOARGS", + """ + PyFrameObject *x = PyEval_GetFrame(); + Py_INCREF(x); + return (PyObject *)x; + """),], prologue="#include \n") + res = module.getframe1() + assert res is sys._getframe(0) diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -151,7 +151,7 @@ if getattr(func, '_elidable_function_', False): raise TypeError("it does not make sense for %s to be both elidable and unroll_safe" % func) if not getattr(func, '_jit_look_inside_', True): - raise TypeError("it does not make sense for %s to be both elidable and dont_look_inside" % func) + raise TypeError("it does not make sense for %s to be both unroll_safe and dont_look_inside" % func) func._jit_unroll_safe_ = True return func diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -233,6 +233,15 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenex(name): + res = rwin32.LoadLibraryExA(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -730,16 +730,21 @@ length = rwin32.MAX_PATH + 1 traits = _preferred_traits(path) win32traits = make_win32_traits(traits) - with traits.scoped_alloc_buffer(length) as buf: - res = win32traits.GetFullPathName( - traits.as_str0(path), rffi.cast(rwin32.DWORD, length), - buf.raw, lltype.nullptr(win32traits.LPSTRP.TO)) - if res == 0: - raise rwin32.lastSavedWindowsError("_getfullpathname failed") - result = buf.str(intmask(res)) - assert result is not None - result = rstring.assert_str0(result) - return result + while True: # should run the loop body maximum twice + with traits.scoped_alloc_buffer(length) as buf: + res = win32traits.GetFullPathName( + traits.as_str0(path), rffi.cast(rwin32.DWORD, length), + buf.raw, lltype.nullptr(win32traits.LPSTRP.TO)) + res = intmask(res) + if res == 0: + raise rwin32.lastSavedWindowsError("_getfullpathname failed") + if res >= length: + length = res + 1 + continue + result = buf.str(res) + assert result is not None + result = rstring.assert_str0(result) + return result c_getcwd = external(UNDERSCORE_ON_WIN32 + 'getcwd', [rffi.CCHARP, rffi.SIZE_T], rffi.CCHARP, diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1775,8 +1775,6 @@ "truncated input", s, pos, size) result.append(res) - if pos > size - unicode_bytes: - break continue t = r_uint(0) h = 0 diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -113,6 +113,7 @@ MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR + LOAD_WITH_ALTERED_SEARCH_PATH """ from rpython.translator.platform import host_factory static_platform = host_factory() @@ -195,6 +196,22 @@ GetModuleHandle = winexternal('GetModuleHandleA', [rffi.CCHARP], HMODULE) LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) + def wrap_loadlibraryex(func): + def loadlibrary(name, handle=None, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + # Requires a full path name with '/' -> '\\' + return func(name, handle, flags) + return loadlibrary + + _LoadLibraryExA = winexternal('LoadLibraryExA', + [rffi.CCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExA = wrap_loadlibraryex(_LoadLibraryExA) + LoadLibraryW = winexternal('LoadLibraryW', [rffi.CWCHARP], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + _LoadLibraryExW = winexternal('LoadLibraryExW', + [rffi.CWCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExW = wrap_loadlibraryex(_LoadLibraryExW) GetProcAddress = winexternal('GetProcAddress', [HMODULE, rffi.CCHARP], rffi.VOIDP) diff --git a/rpython/rlib/test/loadtest/loadtest0.dll b/rpython/rlib/test/loadtest/loadtest0.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9bdcc33a1902f8e989d349c49c2cc08e633aa32b GIT binary patch [cut] diff --git a/rpython/rlib/test/loadtest/loadtest1.dll b/rpython/rlib/test/loadtest/loadtest1.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cb83854875c876717371bdf90488eed9c6571f03 GIT binary patch [cut] diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -83,6 +83,14 @@ # the most intriguing failure of ntpath.py should not repeat, here: assert not data.endswith(stuff) + @win_only + def test__getfullpathname_long(self): + stuff = "C:" + "\\abcd" * 100 + py.test.raises(WindowsError, rposix.getfullpathname, stuff) + ustuff = u"C:" + u"\\abcd" * 100 + res = rposix.getfullpathname(ustuff) + assert res == ustuff + def test_getcwd(self): assert rposix.getcwd() == os.getcwd() diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -6,6 +6,44 @@ from rpython.rlib import rwin32 from rpython.tool.udir import udir +loadtest_dir = os.path.dirname(__file__) + '/loadtest' +test1 = os.path.abspath(loadtest_dir + '/loadtest1.dll') +test0 = os.path.abspath(loadtest_dir + '/loadtest0.dll') + +if not os.path.exists(test1) or not os.path.exists(test0): + # This is how the files, which are checked into the repo, were created + from rpython.translator.tool.cbuild import ExternalCompilationInfo + from rpython.translator.platform import platform + from rpython.translator import cdir + if not os.path.exists(loadtest_dir): + os.mkdir(loadtest_dir) + c_file = udir.ensure("test_rwin32", dir=1).join("test0.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + RPY_EXPORTED + int internal_sum(int a, int b) { + return a + b; + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir]) + lib_name = str(platform.compile([c_file], eci, test0[:-4], + standalone=False)) + assert os.path.abspath(lib_name) == os.path.abspath(test0) + + c_file = udir.ensure("test_rwin32", dir=1).join("test1.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + int internal_sum(int a, int b); + RPY_EXPORTED + int sum(int a, int b) { + return internal_sum(a, b); + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir], + libraries=[loadtest_dir + '/loadtest0']) + lib_name = str(platform.compile([c_file], eci, test1[:-4], + standalone=False, )) + assert os.path.abspath(lib_name) == os.path.abspath(test1) def test_get_osfhandle(): fid = open(str(udir.join('validate_test.txt')), 'w') @@ -28,13 +66,13 @@ "import time;" "time.sleep(10)", ], - ) + ) print proc.pid handle = rwin32.OpenProcess(rwin32.PROCESS_ALL_ACCESS, False, proc.pid) assert rwin32.TerminateProcess(handle, signal.SIGTERM) == 1 rwin32.CloseHandle(handle) assert proc.wait() == signal.SIGTERM - + @py.test.mark.dont_track_allocations('putenv intentionally keeps strings alive') def test_wenviron(): name, value = u'PYPY_TEST_日本', u'foobar日本' @@ -55,3 +93,48 @@ msg = rwin32.FormatErrorW(34) assert type(msg) is unicode assert u'%2' in msg + +def test_loadlibraryA(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibrary(test0) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibrary(test1) + assert not hdll + + assert os.path.exists(test1) + + hdll = rwin32.LoadLibraryExA(test1) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + +def test_loadlibraryW(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibraryW(unicode(test0)) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibraryW(unicode(test1)) + assert not hdll + + assert os.path.exists(unicode(test1)) + + hdll = rwin32.LoadLibraryExW(unicode(test1)) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + +def test_loadlibrary_unicode(): + import shutil + test0u = unicode(udir.join(u'load\u03betest.dll')) + shutil.copyfile(test0, test0u) + hdll = rwin32.LoadLibraryW(test0u) + assert hdll From pypy.commits at gmail.com Mon Oct 8 16:51:20 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 08 Oct 2018 13:51:20 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: typo Message-ID: <5bbbc348.1c69fb81.4a712.0dd7@mx.google.com> Author: Matti Picus Branch: cffi_dlopen_unicode Changeset: r95195:78c40e133e02 Date: 2018-10-08 23:50 +0300 http://bitbucket.org/pypy/pypy/changeset/78c40e133e02/ Log: typo diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -119,7 +119,7 @@ assert type(lib.add42) is _cffi_backend.FFI.CData def test_dlopen_unicode(self): - if getattr(self, extmodU, None): + if getattr(self, 'extmodU', None): import _cffi_backend self.fix_path() from re_python_pysrc import ffi From pypy.commits at gmail.com Tue Oct 9 10:25:40 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 09 Oct 2018 07:25:40 -0700 (PDT) Subject: [pypy-commit] pypy default: fix win32 translation, also remove some MSVC compiler warnings Message-ID: <5bbcba64.1c69fb81.590a8.0d09@mx.google.com> Author: Matti Picus Branch: Changeset: r95196:0bd6514900a7 Date: 2018-10-09 17:23 +0300 http://bitbucket.org/pypy/pypy/changeset/0bd6514900a7/ Log: fix win32 translation, also remove some MSVC compiler warnings diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -95,7 +95,9 @@ if os.name == 'nt': do_includes = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include static void _cffi_init(void); diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1554,6 +1554,9 @@ } ''' separate_module_sources.append(get_pythonapi_source) + kwds['post_include_bits'] = [ + 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', + ] eci = ExternalCompilationInfo( include_dirs=include_dirs, diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -183,7 +183,9 @@ if os.name == 'nt': _source_code = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include #include diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -835,7 +835,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = rwin32.get_osfhandle(fileno) + fh = rffi.cast(HANDLE, rwin32.get_osfhandle(fileno)) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,7 +137,10 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', + 'RPY_EXTERN void* enter_suppress_iph();', + 'RPY_EXTERN void exit_suppress_iph(void* handle);', + ] else: separate_module_sources = [] post_include_bits = [] @@ -235,7 +238,8 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', + 'direct.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -20,7 +20,7 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h', 'stdio.h', 'stdlib.h'], + includes = ['windows.h', 'stdio.h', 'stdlib.h', 'io.h'], libraries = ['kernel32'], ) else: @@ -197,9 +197,9 @@ LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) def wrap_loadlibraryex(func): - def loadlibrary(name, handle=None, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + def loadlibrary(name, flags=LOAD_WITH_ALTERED_SEARCH_PATH): # Requires a full path name with '/' -> '\\' - return func(name, handle, flags) + return func(name, NULL_HANDLE, flags) return loadlibrary _LoadLibraryExA = winexternal('LoadLibraryExA', @@ -217,7 +217,7 @@ rffi.VOIDP) FreeLibrary = winexternal('FreeLibrary', [HMODULE], BOOL, releasegil=False) - LocalFree = winexternal('LocalFree', [HLOCAL], DWORD) + LocalFree = winexternal('LocalFree', [HLOCAL], HLOCAL) CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL, releasegil=False, save_err=rffi.RFFI_SAVE_LASTERROR) CloseHandle_no_err = winexternal('CloseHandle', [HANDLE], BOOL, @@ -232,12 +232,12 @@ [DWORD, rffi.VOIDP, DWORD, DWORD, rffi.CWCHARP, DWORD, rffi.VOIDP], DWORD) - _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) + _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], rffi.INTP) def get_osfhandle(fd): from rpython.rlib.rposix import FdValidator with FdValidator(fd): - handle = _get_osfhandle(fd) + handle = rffi.cast(HANDLE, _get_osfhandle(fd)) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle From pypy.commits at gmail.com Tue Oct 9 12:15:55 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 09 Oct 2018 09:15:55 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5bbcd43b.1c69fb81.96a93.d68c@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95197:fadba769e9bf Date: 2018-10-09 17:36 +0300 http://bitbucket.org/pypy/pypy/changeset/fadba769e9bf/ Log: merge default into branch diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -95,7 +95,9 @@ if os.name == 'nt': do_includes = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include static void _cffi_init(void); diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1557,6 +1557,9 @@ } ''' separate_module_sources.append(get_pythonapi_source) + kwds['post_include_bits'] = [ + 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', + ] eci = ExternalCompilationInfo( include_dirs=include_dirs, diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -231,7 +231,9 @@ if os.name == 'nt': _source_code = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include #include diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -835,7 +835,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = rwin32.get_osfhandle(fileno) + fh = rffi.cast(HANDLE, rwin32.get_osfhandle(fileno)) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,7 +137,10 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', + 'RPY_EXTERN void* enter_suppress_iph();', + 'RPY_EXTERN void exit_suppress_iph(void* handle);', + ] else: separate_module_sources = [] post_include_bits = [] @@ -235,7 +238,8 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', + 'direct.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -20,7 +20,7 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h', 'stdio.h', 'stdlib.h'], + includes = ['windows.h', 'stdio.h', 'stdlib.h', 'io.h'], libraries = ['kernel32'], ) else: @@ -197,9 +197,9 @@ LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) def wrap_loadlibraryex(func): - def loadlibrary(name, handle=None, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + def loadlibrary(name, flags=LOAD_WITH_ALTERED_SEARCH_PATH): # Requires a full path name with '/' -> '\\' - return func(name, handle, flags) + return func(name, NULL_HANDLE, flags) return loadlibrary _LoadLibraryExA = winexternal('LoadLibraryExA', @@ -217,7 +217,7 @@ rffi.VOIDP) FreeLibrary = winexternal('FreeLibrary', [HMODULE], BOOL, releasegil=False) - LocalFree = winexternal('LocalFree', [HLOCAL], DWORD) + LocalFree = winexternal('LocalFree', [HLOCAL], HLOCAL) CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL, releasegil=False, save_err=rffi.RFFI_SAVE_LASTERROR) CloseHandle_no_err = winexternal('CloseHandle', [HANDLE], BOOL, @@ -232,12 +232,12 @@ [DWORD, rffi.VOIDP, DWORD, DWORD, rffi.CWCHARP, DWORD, rffi.VOIDP], DWORD) - _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) + _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], rffi.INTP) def get_osfhandle(fd): from rpython.rlib.rposix import FdValidator with FdValidator(fd): - handle = _get_osfhandle(fd) + handle = rffi.cast(HANDLE, _get_osfhandle(fd)) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle From pypy.commits at gmail.com Wed Oct 10 05:08:03 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 10 Oct 2018 02:08:03 -0700 (PDT) Subject: [pypy-commit] pypy default: move include before declaration Message-ID: <5bbdc173.1c69fb81.f52d3.9008@mx.google.com> Author: Matti Picus Branch: Changeset: r95198:17235f220103 Date: 2018-10-10 12:06 +0300 http://bitbucket.org/pypy/pypy/changeset/17235f220103/ Log: move include before declaration diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1540,7 +1540,6 @@ if sys.platform == 'win32': get_pythonapi_source = ''' - #include RPY_EXTERN HANDLE pypy_get_pythonapi_handle() { MEMORY_BASIC_INFORMATION mi; @@ -1554,7 +1553,7 @@ } ''' separate_module_sources.append(get_pythonapi_source) - kwds['post_include_bits'] = [ + kwds['post_include_bits'] = ['#include ', 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', ] From pypy.commits at gmail.com Wed Oct 10 05:08:05 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 10 Oct 2018 02:08:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5bbdc175.1c69fb81.3864b.e6c9@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95199:d52c3dff08d6 Date: 2018-10-10 12:07 +0300 http://bitbucket.org/pypy/pypy/changeset/d52c3dff08d6/ Log: merge default into branch diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1543,7 +1543,6 @@ if sys.platform == 'win32': get_pythonapi_source = ''' - #include RPY_EXTERN HANDLE pypy_get_pythonapi_handle() { MEMORY_BASIC_INFORMATION mi; @@ -1557,7 +1556,7 @@ } ''' separate_module_sources.append(get_pythonapi_source) - kwds['post_include_bits'] = [ + kwds['post_include_bits'] = ['#include ', 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', ] From pypy.commits at gmail.com Wed Oct 10 08:56:20 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 10 Oct 2018 05:56:20 -0700 (PDT) Subject: [pypy-commit] pypy gc-disable: hg merge default Message-ID: <5bbdf6f4.1c69fb81.d7207.47a2@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95200:f15f2aefb48e Date: 2018-10-10 14:51 +0200 http://bitbucket.org/pypy/pypy/changeset/f15f2aefb48e/ Log: hg merge default diff too long, truncating to 2000 out of 22739 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -33,7 +33,12 @@ 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1 +4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1 +4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1 +d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0 +e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1 +a24d6c7000c8099c73d3660857f7e3cee5ac045c RevDB-pypy2.7-v5.6.2 fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1 diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ Welcome to PyPy! -PyPy is an interperter that implements the Python programming language, based +PyPy is an interpreter that implements the Python programming language, based on the RPython compiler framework for dynamic language implementations. The home page for the interpreter is: @@ -15,29 +15,29 @@ http://doc.pypy.org/ -More documentation about the RPython framework can be found here +More documentation about the RPython framework can be found here: - http://rpython.readthedocs.io + http://rpython.readthedocs.io/ -The source for the documentation is in the pypy/doc directory +The source for the documentation is in the pypy/doc directory. + Using PyPy instead of CPython -============================= +----------------------------- -Please read the information at http://pypy.org to find the correct way to +Please read the information at http://pypy.org/ to find the correct way to download and use PyPy as an alternative to CPython. + Building -======== +-------- Building PyPy is not the recommended way to obtain the PyPy alternative python interpreter. It is time-consuming and requires significant computing resources. -More information can be found here +More information can be found here: http://doc.pypy.org/en/latest/build.html Enjoy and send us feedback! the pypy-dev team - - diff --git a/lib-python/2.7/code.py b/lib-python/2.7/code.py --- a/lib-python/2.7/code.py +++ b/lib-python/2.7/code.py @@ -104,6 +104,12 @@ except SystemExit: raise except: + if softspace(sys.stdout, 0): + print + try: + sys.stdout.flush() + except: + pass self.showtraceback() else: if softspace(sys.stdout, 0): diff --git a/lib-python/2.7/fractions.py b/lib-python/2.7/fractions.py --- a/lib-python/2.7/fractions.py +++ b/lib-python/2.7/fractions.py @@ -517,8 +517,13 @@ # Get integers right. return hash(self._numerator) # Expensive check, but definitely correct. - if self == float(self): - return hash(float(self)) + # PyPy: the following 4 lines used to be almost twice slower: + # if self == float(self): + # return hash(float(self)) + f = float(self) + x, y = f.as_integer_ratio() # signs are correct: y is positive + if self._numerator == x and self._denominator == y: + return hash(f) else: # Use tuple's hash to avoid a high collision rate on # simple fractions. diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -136,9 +136,14 @@ __get_hash = __get_openssl_constructor algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) -except ImportError: +except ImportError as e: new = __py_new __get_hash = __get_builtin_constructor + # added by PyPy + import warnings + warnings.warn("The _hashlib module is not available, falling back " + "to a much slower implementation (%s)" % str(e), + RuntimeWarning) for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py --- a/lib-python/2.7/opcode.py +++ b/lib-python/2.7/opcode.py @@ -194,5 +194,6 @@ def_op('CALL_METHOD', 202) # #args not including 'self' def_op('BUILD_LIST_FROM_ARG', 203) jrel_op('JUMP_IF_NOT_DEBUG', 204) # jump over assert statements +def_op('LOAD_REVDB_VAR', 205) # reverse debugger (syntax example: $5) del def_op, name_op, jrel_op, jabs_op diff --git a/lib-python/2.7/shutil.py b/lib-python/2.7/shutil.py --- a/lib-python/2.7/shutil.py +++ b/lib-python/2.7/shutil.py @@ -396,17 +396,21 @@ return archive_name -def _call_external_zip(base_dir, zip_filename, verbose=False, dry_run=False): +def _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger): # XXX see if we want to keep an external call here if verbose: zipoptions = "-r" else: zipoptions = "-rq" - from distutils.errors import DistutilsExecError - from distutils.spawn import spawn + cmd = ["zip", zipoptions, zip_filename, base_dir] + if logger is not None: + logger.info(' '.join(cmd)) + if dry_run: + return + import subprocess try: - spawn(["zip", zipoptions, zip_filename, base_dir], dry_run=dry_run) - except DistutilsExecError: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: # XXX really should distinguish between "couldn't find # external 'zip' command" and "zip failed". raise ExecError, \ @@ -440,7 +444,7 @@ zipfile = None if zipfile is None: - _call_external_zip(base_dir, zip_filename, verbose, dry_run) + _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger) else: if logger is not None: logger.info("creating '%s' and adding '%s' to it", diff --git a/lib-python/2.7/test/test_inspect.py b/lib-python/2.7/test/test_inspect.py --- a/lib-python/2.7/test/test_inspect.py +++ b/lib-python/2.7/test/test_inspect.py @@ -45,6 +45,9 @@ git = mod.StupidGit() +class ExampleClassWithSlot(object): + __slots__ = 'myslot' + class IsTestBase(unittest.TestCase): predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, inspect.isframe, inspect.isfunction, inspect.ismethod, @@ -96,7 +99,11 @@ else: self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals)) if hasattr(types, 'MemberDescriptorType'): - self.istest(inspect.ismemberdescriptor, 'type(lambda: None).func_globals') + # App-level slots are member descriptors on both PyPy and + # CPython, but the various built-in attributes are all + # getsetdescriptors on PyPy. So check ismemberdescriptor() + # with an app-level slot. + self.istest(inspect.ismemberdescriptor, 'ExampleClassWithSlot.myslot') else: self.assertFalse(inspect.ismemberdescriptor(type(lambda: None).func_globals)) diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py --- a/lib-python/2.7/types.py +++ b/lib-python/2.7/types.py @@ -83,9 +83,19 @@ DictProxyType = type(TypeType.__dict__) NotImplementedType = type(NotImplemented) -# For Jython, the following two types are identical +# +# On CPython, FunctionType.__code__ is a 'getset_descriptor', but +# FunctionType.__globals__ is a 'member_descriptor', just like app-level +# slots. On PyPy, all descriptors of built-in types are +# 'getset_descriptor', but the app-level slots are 'member_descriptor' +# as well. (On Jython the situation might still be different.) +# +# Note that MemberDescriptorType was equal to GetSetDescriptorType in +# PyPy <= 6.0. +# GetSetDescriptorType = type(FunctionType.func_code) -MemberDescriptorType = type(FunctionType.func_globals) +class _C(object): __slots__ = 's' +MemberDescriptorType = type(_C.s) del sys, _f, _g, _C, _x # Not for export diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -183,6 +183,7 @@ self._buffer = self._ffiarray(self._length_, autofree=True) for i, arg in enumerate(args): self[i] = arg + _init_no_arg_ = __init__ def _fix_index(self, index): if index < 0: diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -120,7 +120,7 @@ raise ValueError( "Buffer size too small (%d instead of at least %d bytes)" % (len(buf) + offset, size + offset)) - result = self() + result = self._newowninstance_() dest = result._buffer.buffer try: raw_addr = buf._pypy_raw_address() @@ -131,6 +131,11 @@ memmove(dest, raw_addr, size) return result + def _newowninstance_(self): + result = self.__new__(self) + result._init_no_arg_() + return result + class CArgObject(object): """ simple wrapper around buffer, just for the case of freeing @@ -162,6 +167,7 @@ def __init__(self, *args, **kwds): raise TypeError("%s has no type" % (type(self),)) + _init_no_arg_ = __init__ def _ensure_objects(self): if '_objects' not in self.__dict__: diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -268,6 +268,7 @@ return raise TypeError("Unknown constructor %s" % (args,)) + _init_no_arg_ = __init__ def _wrap_callable(self, to_call, argtypes): def f(*args): @@ -485,6 +486,8 @@ return cobj, cobj._to_ffi_param(), type(cobj) def _convert_args_for_callback(self, argtypes, args): + from _ctypes.structure import StructOrUnion + # assert len(argtypes) == len(args) newargs = [] for argtype, arg in zip(argtypes, args): @@ -494,6 +497,10 @@ param = param._get_buffer_value() elif self._is_primitive(argtype): param = param.value + elif isinstance(param, StructOrUnion): # not a *pointer* to struct + newparam = StructOrUnion.__new__(type(param)) + param._copy_to(newparam._buffer.buffer) + param = newparam newargs.append(param) return newargs @@ -557,7 +564,7 @@ keepalive, newarg, newargtype = self._conv_param(argtype, defval) else: import ctypes - val = argtype._type_() + val = argtype._type_._newowninstance_() keepalive = None newarg = ctypes.byref(val) newargtype = type(newarg) diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -67,8 +67,11 @@ self._buffer = ffiarray(1, autofree=True) if value is not None: self.contents = value + def _init_no_arg_(self): + self._buffer = ffiarray(1, autofree=True) self._ffiarray = ffiarray self.__init__ = __init__ + self._init_no_arg_ = _init_no_arg_ self._type_ = TP def _build_ffiargtype(self): @@ -137,27 +140,21 @@ if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): raise TypeError("cast() argument 2 must be a pointer type, not %s" % (tp,)) + result = tp._newowninstance_() if isinstance(obj, (int, long)): - result = tp() result._buffer[0] = obj return result elif obj is None: - result = tp() return result elif isinstance(obj, Array): - ptr = tp.__new__(tp) - ptr._buffer = tp._ffiarray(1, autofree=True) - ptr._buffer[0] = obj._buffer - result = ptr + result._buffer[0] = obj._buffer elif isinstance(obj, bytes): - result = tp() result._buffer[0] = buffer(obj)._pypy_raw_address() return result elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) else: - result = tp() result._buffer[0] = obj._buffer[0] # The casted objects '_objects' member: diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -390,11 +390,14 @@ self._buffer = self._ffiarray(1, autofree=True) if value is not DEFAULT_VALUE: self.value = value + _init_no_arg_ = __init__ def _ensure_objects(self): - if self._type_ not in 'zZP': - assert self._objects is None - return self._objects + # No '_objects' is the common case for primitives. Examples + # where there is an _objects is if _type in 'zZP', or if + # self comes from 'from_buffer(buf)'. See module/test_lib_pypy/ + # ctypes_test/test_buffers.py: test_from_buffer_keepalive. + return getattr(self, '_objects', None) def _getvalue(self): return self._buffer[0] diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -281,6 +281,7 @@ self.__setattr__(name, arg) for name, arg in kwds.items(): self.__setattr__(name, arg) + _init_no_arg_ = __init__ def _subarray(self, fieldtype, name): """Return a _rawffi array of length 1 whose address is the same as diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.5 +Version: 1.12.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.5" -__version_info__ = (1, 11, 5) +__version__ = "1.12.0" +__version_info__ = (1, 12, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h --- a/lib_pypy/cffi/_cffi_errors.h +++ b/lib_pypy/cffi/_cffi_errors.h @@ -50,7 +50,9 @@ "import sys\n" "class FileLike:\n" " def write(self, x):\n" - " of.write(x)\n" + " try:\n" + " of.write(x)\n" + " except: pass\n" " self.buf += x\n" "fl = FileLike()\n" "fl.buf = ''\n" diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,20 +8,43 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350 is still open: on Windows, the code here causes it to link - with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was - attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv - does not make PYTHON3.DLL available, and so the "correctly" compiled - version would not run inside a virtualenv. We will re-apply the fix - after virtualenv has been fixed for some time. For explanation, see - issue #355. For a workaround if you want PYTHON3.DLL and don't worry - about virtualenv, see issue #350. See also 'py_limited_api' in - setuptools_ext.py. + The implementation is messy (issue #350): on Windows, with _MSC_VER, + we have to define Py_LIMITED_API even before including pyconfig.h. + In that case, we guess what pyconfig.h will do to the macros above, + and check our guess after the #include. + + Note that on Windows, with CPython 3.x, you need virtualenv version + >= 16.0.0. Older versions don't copy PYTHON3.DLL. As a workaround + you can remove the definition of Py_LIMITED_API here. + + See also 'py_limited_api' in cffi/setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -221,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.5" + "\ncompiled with cffi version: 1.12.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -96,18 +96,21 @@ self.CData, self.CType = backend._get_types() self.buffer = backend.buffer - def cdef(self, csource, override=False, packed=False): + def cdef(self, csource, override=False, packed=False, pack=None): """Parse the given C source. This registers all declared functions, types, and global variables. The functions and global variables can then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'. The types can be used in 'ffi.new()' and other functions. If 'packed' is specified as True, all structs declared inside this cdef are packed, i.e. laid out without any field alignment at all. + Alternatively, 'pack' can be a small integer, and requests for + alignment greater than that are ignored (pack=1 is equivalent to + packed=True). """ - self._cdef(csource, override=override, packed=packed) + self._cdef(csource, override=override, packed=packed, pack=pack) - def embedding_api(self, csource, packed=False): - self._cdef(csource, packed=packed, dllexport=True) + def embedding_api(self, csource, packed=False, pack=None): + self._cdef(csource, packed=packed, pack=pack, dllexport=True) if self._embedding is None: self._embedding = '' diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -636,6 +636,10 @@ if isinstance(init, bytes): init = [init[i:i+1] for i in range(len(init))] else: + if isinstance(init, CTypesGenericArray): + if (len(init) != len(blob) or + not isinstance(init, CTypesArray)): + raise TypeError("length/type mismatch: %s" % (init,)) init = tuple(init) if len(init) > len(blob): raise IndexError("too many initializers") @@ -730,7 +734,8 @@ return self._new_struct_or_union('union', name, ctypes.Union) def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp, - totalsize=-1, totalalignment=-1, sflags=0): + totalsize=-1, totalalignment=-1, sflags=0, + pack=0): if totalsize >= 0 or totalalignment >= 0: raise NotImplementedError("the ctypes backend of CFFI does not support " "structures completed by verify(); please " @@ -751,6 +756,8 @@ bfield_types[fname] = Ellipsis if sflags & 8: struct_or_union._pack_ = 1 + elif pack: + struct_or_union._pack_ = pack struct_or_union._fields_ = cfields CTypesStructOrUnion._bfield_types = bfield_types # diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -306,11 +306,25 @@ msg = 'parse error\n%s' % (msg,) raise CDefError(msg) - def parse(self, csource, override=False, packed=False, dllexport=False): + def parse(self, csource, override=False, packed=False, pack=None, + dllexport=False): + if packed: + if packed != True: + raise ValueError("'packed' should be False or True; use " + "'pack' to give another value") + if pack: + raise ValueError("cannot give both 'pack' and 'packed'") + pack = 1 + elif pack: + if pack & (pack - 1): + raise ValueError("'pack' must be a power of two, not %r" % + (pack,)) + else: + pack = 0 prev_options = self._options try: self._options = {'override': override, - 'packed': packed, + 'packed': pack, 'dllexport': dllexport} self._internal_parse(csource) finally: diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -342,7 +342,7 @@ fixedlayout = None completed = 0 partial = False - packed = False + packed = 0 def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None): self.name = name @@ -414,11 +414,14 @@ fldtypes = [tp.get_cached_btype(ffi, finishlist) for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) - sflags = 0 + extra_flags = () if self.packed: - sflags = 8 # SF_PACKED + if self.packed == 1: + extra_flags = (8,) # SF_PACKED + else: + extra_flags = (0, self.packed) ffi._backend.complete_struct_or_union(BType, lst, self, - -1, -1, sflags) + -1, -1, *extra_flags) # else: fldtypes = [] diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -893,6 +893,12 @@ else: flags.append("_CFFI_F_CHECK_FIELDS") if tp.packed: + if tp.packed > 1: + raise NotImplementedError( + "%r is declared with 'pack=%r'; only 0 or 1 are " + "supported in API mode (try to use \"...;\", which " + "does not require a 'pack' declaration)" % + (tp, tp.packed)) flags.append("_CFFI_F_PACKED") else: flags.append("_CFFI_F_EXTERNAL") diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,13 +81,8 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) - - On Windows, it's better not to use py_limited_api until issue #355 - can be resolved (by having virtualenv copy PYTHON3.DLL). See also - the start of _cffi_include.h. """ - if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') - and sys.platform != 'win32'): + if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) @@ -167,6 +162,17 @@ module_path = module_name.split('.') module_path[-1] += '.py' generate_mod(os.path.join(self.build_lib, *module_path)) + def get_source_files(self): + # This is called from 'setup.py sdist' only. Exclude + # the generate .py module in this case. + saved_py_modules = self.py_modules + try: + if saved_py_modules: + self.py_modules = [m for m in saved_py_modules + if m != module_name] + return base_class.get_source_files(self) + finally: + self.py_modules = saved_py_modules dist.cmdclass['build_py'] = build_py_make_mod # distutils and setuptools have no notion I could find of a @@ -176,6 +182,7 @@ # the module. So we add it here, which gives a few apparently # harmless warnings about not finding the file outside the # build directory. + # Then we need to hack more in get_source_files(); see above. if dist.py_modules is None: dist.py_modules = [] dist.py_modules.append(module_name) diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py --- a/lib_pypy/grp.py +++ b/lib_pypy/grp.py @@ -4,6 +4,8 @@ from _pwdgrp_cffi import ffi, lib import _structseq +import thread +_lock = thread.allocate_lock() try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f @@ -33,32 +35,35 @@ @builtinify def getgrgid(gid): - res = lib.getgrgid(gid) - if not res: - # XXX maybe check error eventually - raise KeyError(gid) - return _group_from_gstruct(res) + with _lock: + res = lib.getgrgid(gid) + if not res: + # XXX maybe check error eventually + raise KeyError(gid) + return _group_from_gstruct(res) @builtinify def getgrnam(name): if not isinstance(name, basestring): raise TypeError("expected string") name = str(name) - res = lib.getgrnam(name) - if not res: - raise KeyError("'getgrnam(): name not found: %s'" % name) - return _group_from_gstruct(res) + with _lock: + res = lib.getgrnam(name) + if not res: + raise KeyError("'getgrnam(): name not found: %s'" % name) + return _group_from_gstruct(res) @builtinify def getgrall(): - lib.setgrent() lst = [] - while 1: - p = lib.getgrent() - if not p: - break - lst.append(_group_from_gstruct(p)) - lib.endgrent() + with _lock: + lib.setgrent() + while 1: + p = lib.getgrent() + if not p: + break + lst.append(_group_from_gstruct(p)) + lib.endgrent() return lst __all__ = ('struct_group', 'getgrgid', 'getgrnam', 'getgrall') diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py --- a/lib_pypy/pwd.py +++ b/lib_pypy/pwd.py @@ -12,6 +12,8 @@ from _pwdgrp_cffi import ffi, lib import _structseq +import thread +_lock = thread.allocate_lock() try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f @@ -55,10 +57,11 @@ Return the password database entry for the given numeric user ID. See pwd.__doc__ for more on password database entries. """ - pw = lib.getpwuid(uid) - if not pw: - raise KeyError("getpwuid(): uid not found: %s" % uid) - return _mkpwent(pw) + with _lock: + pw = lib.getpwuid(uid) + if not pw: + raise KeyError("getpwuid(): uid not found: %s" % uid) + return _mkpwent(pw) @builtinify def getpwnam(name): @@ -71,10 +74,11 @@ if not isinstance(name, basestring): raise TypeError("expected string") name = str(name) - pw = lib.getpwnam(name) - if not pw: - raise KeyError("getpwname(): name not found: %s" % name) - return _mkpwent(pw) + with _lock: + pw = lib.getpwnam(name) + if not pw: + raise KeyError("getpwname(): name not found: %s" % name) + return _mkpwent(pw) @builtinify def getpwall(): @@ -84,13 +88,14 @@ See pwd.__doc__ for more on password database entries. """ users = [] - lib.setpwent() - while True: - pw = lib.getpwent() - if not pw: - break - users.append(_mkpwent(pw)) - lib.endpwent() + with _lock: + lib.setpwent() + while True: + pw = lib.getpwent() + if not pw: + break + users.append(_mkpwent(pw)) + lib.endpwent() return users __all__ = ('struct_passwd', 'getpwuid', 'getpwnam', 'getpwall') diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -39,14 +39,10 @@ "_csv", "_cppyy", "_pypyjson", "_jitlog" ]) -from rpython.jit.backend import detect_cpu -try: - if detect_cpu.autodetect().startswith('x86'): - if not sys.platform.startswith('openbsd'): - working_modules.add('_vmprof') - working_modules.add('faulthandler') -except detect_cpu.ProcessorAutodetectError: - pass +import rpython.rlib.rvmprof.cintf +if rpython.rlib.rvmprof.cintf.IS_SUPPORTED: + working_modules.add('_vmprof') + working_modules.add('faulthandler') translation_modules = default_modules.copy() translation_modules.update([ @@ -57,6 +53,11 @@ "termios", "_minimal_curses", ]) +reverse_debugger_disable_modules = set([ + "_continuation", "_vmprof", "_multiprocessing", + "micronumpy", + ]) + # XXX this should move somewhere else, maybe to platform ("is this posixish" # check or something) if sys.platform == "win32": @@ -292,6 +293,9 @@ modules = working_modules.copy() if config.translation.sandbox: modules = default_modules + if config.translation.reverse_debugger: + for mod in reverse_debugger_disable_modules: + setattr(config.objspace.usemodules, mod, False) # ignore names from 'essential_modules', notably 'exceptions', which # may not be present in config.objspace.usemodules at all modules = [name for name in modules if name not in essential_modules] @@ -310,3 +314,4 @@ parser = to_optparse(config) #, useoptions=["translation.*"]) option, args = parser.parse_args() print config + print working_modules diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt new file mode 100644 diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt new file mode 100644 diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt new file mode 100644 diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt new file mode 100644 diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt new file mode 100644 diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt new file mode 100644 diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt new file mode 100644 diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt new file mode 100644 diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt new file mode 100644 diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -323,7 +323,8 @@ ------------- * Hash randomization (``-R``) `is ignored in PyPy`_. In CPython - before 3.4 it has `little point`_. + before 3.4 it has `little point`_. Both CPython >= 3.4 and PyPy3 + implement the randomized SipHash algorithm and ignore ``-R``. * You can't store non-string keys in type objects. For example:: diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst --- a/pypy/doc/install.rst +++ b/pypy/doc/install.rst @@ -20,7 +20,7 @@ OS and architecture. You may be able to use either use the `most recent release`_ or one of our `development nightly build`_. These builds depend on dynamically linked libraries that may not be available on your -OS. See the section about `Linux binaries` for more info and alternatives that +OS. See the section about `Linux binaries`_ for more info and alternatives that may work on your system. Please note that the nightly builds are not diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst --- a/pypy/doc/sandbox.rst +++ b/pypy/doc/sandbox.rst @@ -3,6 +3,11 @@ PyPy's sandboxing features ========================== +.. warning:: This is not actively maintained. You will likely have to fix + some issues yourself, or otherwise play around on your own. We provide + this documentation for historical reasions, it will not translate or + run on the latest PyPy code base. + Introduction ------------ diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -18,12 +18,13 @@ 'Antonio Cuni': ['antocuni', 'anto'], 'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'], 'Maciej Fijalkowski': ['fijal'], - 'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'], + 'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf', 'cbolz'], 'Samuele Pedroni': ['pedronis', 'samuele', 'samule'], - 'Richard Plangger':['planrich'], - 'Michael Hudson': ['mwh'], + 'Richard Plangger': ['planrich', 'plan_rich'], + 'Remi Meier': ['remi'], + 'Michael Hudson-Doyle': ['mwh', 'Michael Hudson'], 'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'], - "Amaury Forgeot d'Arc": ['afa'], + "Amaury Forgeot d'Arc": ['afa', 'amauryfa at gmail.com'], 'Alex Gaynor': ['alex', 'agaynor'], 'David Schneider': ['bivab', 'david'], 'Christian Tismer': ['chris', 'christian', 'tismer', @@ -41,7 +42,7 @@ 'Mark Pearse': ['mwp'], 'Toon Verwaest': ['tverwaes'], 'Eric van Riet Paap': ['ericvrp'], - 'Jacob Hallen': ['jacob', 'jakob'], + 'Jacob Hallen': ['jacob', 'jakob', 'jacob hallen'], 'Anders Lehmann': ['ale', 'anders'], 'Bert Freudenberg': ['bert'], 'Boris Feigin': ['boris', 'boria'], @@ -69,19 +70,25 @@ 'Manuel Jacob': ['mjacob'], 'Rami Chowdhury': ['necaris'], 'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'], - 'Wenzhu Man':['wenzhu man', 'wenzhuman'], - 'Anton Gulenko':['anton gulenko', 'anton_gulenko'], - 'Richard Lancaster':['richardlancaster'], - 'William Leslie':['William ML Leslie'], - 'Spenser Bauman':['Spenser Andrew Bauman'], - 'Raffael Tfirst':['raffael.tfirst at gmail.com'], - 'timo':['timo at eistee.fritz.box'], - 'Jasper Schulz':['Jasper.Schulz', 'jbs'], - 'Aaron Gallagher':['"Aaron Gallagher'], - 'Yasir Suhail':['yasirs'], + 'Wenzhu Man': ['wenzhu man', 'wenzhuman'], + 'Anton Gulenko': ['anton gulenko', 'anton_gulenko'], + 'Richard Lancaster': ['richardlancaster'], + 'William Leslie': ['William ML Leslie'], + 'Spenser Bauman': ['Spenser Andrew Bauman'], + 'Raffael Tfirst': ['raffael.tfirst at gmail.com'], + 'timo': ['timo at eistee.fritz.box'], + 'Jasper Schulz': ['Jasper.Schulz', 'jbs'], + 'Aaron Gallagher': ['"Aaron Gallagher'], + 'Yasir Suhail': ['yasirs'], 'Squeaky': ['squeaky'], - "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'], "Dodan Mihai": ['mihai.dodan at gmail.com'], + 'Wim Lavrijsen': ['wlav'], + 'Toon Verwaest': ['toon', 'tverwaes'], + 'Seo Sanghyeon': ['sanxiyn'], + 'Leonardo Santagada': ['santagada'], + 'Laurence Tratt': ['ltratt'], + 'Pieter Zieschang': ['pzieschang', 'p_zieschang at yahoo.de'], + 'John Witulski': ['witulski'], } alias_map = {} @@ -103,7 +110,8 @@ return set() ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks', 'in-progress', 'bits of', 'even a little', 'floating', - 'a bit', 'reviewing'] + 'a bit', 'reviewing', 'looking', 'advising', 'partly', 'ish', + 'watching', 'mostly', 'jumping'] sep_words = ['and', ';', '+', '/', 'with special by'] nicknames = match.group(1) for word in ignore_words: diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -7,9 +7,13 @@ .. branch: cppyy-packaging -Upgrade to backend 0.6.0, support exception handling from wrapped functions, -update enum handling, const correctness for data members and associated tests, -support anonymous enums, support for function pointer arguments +Main items: vastly better template resolution and improved performance. In +detail: upgrade to backend 1.4, improved handling of templated methods and +functions (in particular automatic deduction of types), improved pythonization +interface, range of compatibility fixes for Python3, free functions now take +fast libffi path when possible, moves for strings (incl. from Python str), +easier/faster handling of std::vector by numpy, improved and faster object +identity preservation .. branch: socket_default_timeout_blockingness @@ -18,3 +22,20 @@ .. branch: crypt_h Include crypt.h for crypt() on Linux + +.. branch: gc-more-logging + +Log additional gc-minor and gc-collect-step info in the PYPYLOG + +.. branch: reverse-debugger + +The reverse-debugger branch has been merged. For more information, see +https://bitbucket.org/pypy/revdb + + +.. branch: pyparser-improvements-3 + +Small refactorings in the Python parser. + +.. branch: fix-readme-typo + diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -29,29 +29,28 @@ ``C:\Users\\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python`` or in ``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``. -A current version of ``setuptools`` will be able to find it there. For -Windows 10, you must right-click the download, and under ``Properties`` -> -``Compatibility`` mark it as ``Run run this program in comatibility mode for`` -``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``, +A current version of ``setuptools`` will be able to find it there. +Also, you must download and install the ``.Net Framework 3.5``, otherwise ``mt.exe`` will silently fail. Installation will begin automatically by running the mt.exe command by hand from a DOS window (that is how the author discovered the problem). .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/EN-US/DOWNLOAD/DETAILS.ASPX?ID=44266 -Installing "Build Tools for Visual Studio 2017" (for Python 3) +Installing "Build Tools for Visual Studio 2015" (for Python 3) -------------------------------------------------------------- -As documented in the CPython Wiki_, CPython now recommends Visual C++ version -14.0. A compact version of the compiler suite can be obtained from Microsoft_ -downloads, search the page for "Build Tools for Visual Studio 2017". +As documented in the CPython Wiki_, CPython recommends Visual C++ version +14.0 for python version 3.5. A compact version of the compiler suite can be +obtained from Microsoft_ downloads, search the page for "Microsoft Build Tools 2015". -You will also need to install the the `Windows SDK`_ in order to use the -`mt.exe` mainfest compiler. +You will need to reboot the computer for the installation to successfully install and +run the `mt.exe` mainfest compiler. The installation will set the +`VS140COMNTOOLS` environment variable, this is key to distutils/setuptools +finding the compiler .. _Wiki: https://wiki.python.org/moin/WindowsCompilers -.. _Microsoft: https://www.visualstudio.com/downloads -.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk +.. _Microsoft: https://www.visualstudio.com/vs/older-downloads/ Translating PyPy with Visual Studio ----------------------------------- @@ -99,6 +98,9 @@ Setting Up Visual Studio 9.0 for building SSL in Python3 -------------------------------------------------------- +**Note: this is old information, left for historical reference. We recommend +using Visual Studio 2015, which now seems to properly set this all up.** + On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C compiler, and ``cffi`` depends on ``distutils`` to find the compiler. The @@ -146,14 +148,14 @@ Installing external packages ---------------------------- -We uses a `repository` parallel to pypy to hold binary compiled versions of the +We uses a subrepository_ inside pypy to hold binary compiled versions of the build dependencies for windows. As part of the `rpython` setup stage, environment variables will be set to use these dependencies. The repository has a README file on how to replicate, and a branch for each supported platform. You may run the `get_externals.py` utility to checkout the proper branch for your platform and PyPy version. -.. _repository: https://bitbucket.org/pypy/external +.. _subrepository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -83,11 +83,24 @@ sys.excepthook(), catching SystemExit, printing a newline after sys.stdout if needed, etc. """ + # don't use try:except: here, otherwise the exception remains + # visible in user code. Make sure revdb_stop is a callable, so + # that we can call it immediately after finally: below. Doing + # so minimizes the number of "blind" lines that we need to go + # back from, with "bstep", after we do "continue" in revdb. + if '__pypy__' in sys.builtin_module_names: + from __pypy__ import revdb_stop + else: + revdb_stop = None + if revdb_stop is None: + revdb_stop = lambda: None + try: # run it try: f(*fargs, **fkwds) finally: + revdb_stop() sys.settrace(None) sys.setprofile(None) diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -673,6 +673,7 @@ ops.JUMP_IF_NOT_DEBUG: 0, ops.BUILD_LIST_FROM_ARG: 1, + ops.LOAD_REVDB_VAR: 1, } diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -1593,6 +1593,8 @@ return Num.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Str): return Str.from_object(space, w_node) + if space.isinstance_w(w_node, get(space).w_RevDBMetaVar): + return RevDBMetaVar.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Attribute): return Attribute.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_Subscript): @@ -2456,6 +2458,41 @@ State.ast_type('Str', 'expr', ['s']) +class RevDBMetaVar(expr): + + def __init__(self, metavar, lineno, col_offset): + self.metavar = metavar + expr.__init__(self, lineno, col_offset) + + def walkabout(self, visitor): + visitor.visit_RevDBMetaVar(self) + + def mutate_over(self, visitor): + return visitor.visit_RevDBMetaVar(self) + + def to_object(self, space): + w_node = space.call_function(get(space).w_RevDBMetaVar) + w_metavar = space.newint(self.metavar) # int + space.setattr(w_node, space.newtext('metavar'), w_metavar) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) + return w_node + + @staticmethod + def from_object(space, w_node): + w_metavar = get_field(space, w_node, 'metavar', False) + w_lineno = get_field(space, w_node, 'lineno', False) + w_col_offset = get_field(space, w_node, 'col_offset', False) + _metavar = space.int_w(w_metavar) + _lineno = space.int_w(w_lineno) + _col_offset = space.int_w(w_col_offset) + return RevDBMetaVar(_metavar, _lineno, _col_offset) + +State.ast_type('RevDBMetaVar', 'expr', ['metavar']) + + class Attribute(expr): def __init__(self, value, attr, ctx, lineno, col_offset): @@ -3588,6 +3625,8 @@ return self.default_visitor(node) def visit_Str(self, node): return self.default_visitor(node) + def visit_RevDBMetaVar(self, node): + return self.default_visitor(node) def visit_Attribute(self, node): return self.default_visitor(node) def visit_Subscript(self, node): @@ -3804,6 +3843,9 @@ def visit_Str(self, node): pass + def visit_RevDBMetaVar(self, node): + pass + def visit_Attribute(self, node): node.value.walkabout(self) diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1165,6 +1165,11 @@ elif first_child_type == tokens.BACKQUOTE: expr = self.handle_testlist(atom_node.get_child(1)) return ast.Repr(expr, atom_node.get_lineno(), atom_node.get_column()) + elif first_child_type == tokens.REVDBMETAVAR: + string = atom_node.get_child(0).get_value() + return ast.RevDBMetaVar(int(string[1:]), + atom_node.get_lineno(), + atom_node.get_column()) else: raise AssertionError("unknown atom") diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -1203,6 +1203,20 @@ sub.value.walkabout(self) self._compile_slice(sub.slice, sub.ctx) + def _revdb_metavar(self, node): + # moved in its own function for the import statement + from pypy.interpreter.reverse_debugging import dbstate + if not dbstate.standard_code: + self.emit_op_arg(ops.LOAD_REVDB_VAR, node.metavar) + return True + return False + + def visit_RevDBMetaVar(self, node): + if self.space.reverse_debugging and self._revdb_metavar(node): + return + self.error("Unknown character ('$NUM' is only valid in the " + "reverse-debugger)", node) + class TopLevelCodeGenerator(PythonCodeGenerator): diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1096,7 +1096,7 @@ s = self.get_first_expr("'hi' ' implicitly' ' extra'") assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap("hi implicitly extra")) - sentence = u"Die Männer ärgen sich!" + sentence = u"Die Männer ärgern sich!" source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,) info = pyparse.CompileInfo("", "exec") tree = self.parser.parse_source(source.encode("utf-7"), info) diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -27,7 +27,7 @@ generator._resolve_block_targets(blocks) return generator, blocks -class TestCompiler: +class BaseTestCompiler: """These tests compile snippets of code and check them by running them with our own interpreter. These are thus not completely *unit* tests, but given that our interpreter is @@ -74,6 +74,9 @@ def error_test(self, source, exc_type): py.test.raises(exc_type, self.simple_test, source, None, None) + +class TestCompiler(BaseTestCompiler): + def test_issue_713(self): func = "def f(_=2): return (_ if _ else _) if False else _" yield self.st, func, "f()", 2 @@ -953,6 +956,22 @@ yield (self.st, "x=(lambda: (-0.0, 0.0), lambda: (0.0, -0.0))[1]()", 'repr(x)', '(0.0, -0.0)') +class TestCompilerRevDB(BaseTestCompiler): + spaceconfig = {"translation.reverse_debugger": True} + + def test_revdb_metavar(self): + from pypy.interpreter.reverse_debugging import dbstate, setup_revdb + self.space.reverse_debugging = True + try: + setup_revdb(self.space) + dbstate.standard_code = False + dbstate.metavars = [self.space.wrap(6)] + self.simple_test("x = 7*$0", "x", 42) + dbstate.standard_code = True + self.error_test("x = 7*$0", SyntaxError) + finally: + self.space.reverse_debugging = False + class AppTestCompiler: diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl --- a/pypy/interpreter/astcompiler/tools/Python.asdl +++ b/pypy/interpreter/astcompiler/tools/Python.asdl @@ -71,6 +71,7 @@ | Repr(expr value) | Num(object n) -- a number as a PyObject. | Str(string s) -- need to specify raw, unicode, etc? + | RevDBMetaVar(int metavar) -- other literals? bools? -- the following expression can appear in assignment context diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -430,6 +430,8 @@ """Base class for the interpreter-level implementations of object spaces. http://pypy.readthedocs.org/en/latest/objspace.html""" + reverse_debugging = False + @not_rpython def __init__(self, config=None): "Basic initialization of objects." @@ -441,6 +443,7 @@ from pypy.config.pypyoption import get_pypy_config config = get_pypy_config(translating=False) self.config = config + self.reverse_debugging = config.translation.reverse_debugger self.builtin_modules = {} self.reloading_modules = {} @@ -458,6 +461,9 @@ def startup(self): # To be called before using the space + if self.reverse_debugging: + self._revdb_startup() + self.threadlocals.enter_thread(self) # Initialize already imported builtin modules @@ -868,7 +874,8 @@ w_s1 = self.interned_strings.get(s) if w_s1 is None: w_s1 = w_s - self.interned_strings.set(s, w_s1) + if self._side_effects_ok(): + self.interned_strings.set(s, w_s1) return w_s1 def new_interned_str(self, s): @@ -878,9 +885,39 @@ w_s1 = self.interned_strings.get(s) if w_s1 is None: w_s1 = self.newtext(s) - self.interned_strings.set(s, w_s1) + if self._side_effects_ok(): + self.interned_strings.set(s, w_s1) return w_s1 + def _revdb_startup(self): + # moved in its own function for the import statement + from pypy.interpreter.reverse_debugging import setup_revdb + setup_revdb(self) + + def _revdb_standard_code(self): + # moved in its own function for the import statement + from pypy.interpreter.reverse_debugging import dbstate + return dbstate.standard_code + + def _side_effects_ok(self): + # For the reverse debugger: we run compiled watchpoint + # expressions in a fast way that will crash if they have + # side-effects. The obvious Python code with side-effects is + # documented "don't do that"; but some non-obvious side + # effects are also common, like interning strings (from + # unmarshalling the code object containing the watchpoint + # expression) to the two attribute caches in mapdict.py and + # typeobject.py. For now, we have to identify such places + # that are not acceptable for "reasonable" read-only + # watchpoint expressions, and write: + # + # if not space._side_effects_ok(): + # don't cache. + # + if self.reverse_debugging: + return self._revdb_standard_code() + return True + def is_interned_str(self, s): # interface for marshal_impl if not we_are_translated(): diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -64,6 +64,8 @@ return frame def enter(self, frame): + if self.space.reverse_debugging: + self._revdb_enter(frame) frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) @@ -84,6 +86,8 @@ # be accessed also later frame_vref() jit.virtual_ref_finish(frame_vref, frame) + if self.space.reverse_debugging: + self._revdb_leave(got_exception) # ________________________________________________________________ @@ -153,6 +157,8 @@ Like bytecode_trace() but doesn't invoke any other events besides the trace function. """ + if self.space.reverse_debugging: + self._revdb_potential_stop_point(frame) if (frame.get_w_f_trace() is None or self.is_tracing or self.gettrace() is None): return @@ -385,6 +391,21 @@ if self.space.check_signal_action is not None: self.space.check_signal_action.perform(self, None) + def _revdb_enter(self, frame): + # moved in its own function for the import statement + from pypy.interpreter.reverse_debugging import enter_call + enter_call(self.topframeref(), frame) + + def _revdb_leave(self, got_exception): + # moved in its own function for the import statement + from pypy.interpreter.reverse_debugging import leave_call + leave_call(self.topframeref(), got_exception) + + def _revdb_potential_stop_point(self, frame): + # moved in its own function for the import statement + from pypy.interpreter.reverse_debugging import potential_stop_point + potential_stop_point(frame) + def _freeze_(self): raise Exception("ExecutionContext instances should not be seen during" " translation. Now is a good time to inspect the" diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -832,9 +832,11 @@ def fget_f_builtins(self, space): return self.get_builtin().getdict(space) + def get_f_back(self): + return ExecutionContext.getnextframe_nohidden(self) + def fget_f_back(self, space): - f_back = ExecutionContext.getnextframe_nohidden(self) - return f_back + return self.get_f_back() def fget_f_lasti(self, space): return self.space.newint(self.last_instr) diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -437,6 +437,8 @@ self.WITH_CLEANUP(oparg, next_instr) elif opcode == opcodedesc.YIELD_VALUE.index: self.YIELD_VALUE(oparg, next_instr) + elif opcode == opcodedesc.LOAD_REVDB_VAR.index: + self.LOAD_REVDB_VAR(oparg, next_instr) else: self.MISSING_OPCODE(oparg, next_instr) @@ -1047,9 +1049,16 @@ def YIELD_VALUE(self, oparg, next_instr): raise Yield + def _revdb_jump_backward(self, jumpto): + # moved in its own function for the import statement + from pypy.interpreter.reverse_debugging import jump_backward + jump_backward(self, jumpto) + def jump_absolute(self, jumpto, ec): # this function is overridden by pypy.module.pypyjit.interp_jit check_nonneg(jumpto) + if self.space.reverse_debugging: + self._revdb_jump_backward(jumpto) return jumpto def JUMP_FORWARD(self, jumpby, next_instr): @@ -1303,6 +1312,18 @@ w_dict = self.peekvalue() self.space.setitem(w_dict, w_key, w_value) + def _revdb_load_var(self, oparg): + # moved in its own function for the import statement + from pypy.interpreter.reverse_debugging import load_metavar + w_var = load_metavar(oparg) + self.pushvalue(w_var) + + def LOAD_REVDB_VAR(self, oparg, next_instr): + if self.space.reverse_debugging: + self._revdb_load_var(oparg) + else: + self.MISSING_OPCODE(oparg, next_instr) + ### ____________________________________________________________ ### @@ -1607,7 +1628,7 @@ else: skip_leading_underscores = False for name in all: - if skip_leading_underscores and name[0]=='_': + if skip_leading_underscores and name and name[0] == '_': continue into_locals[name] = getattr(module, name) ''', filename=__file__) diff --git a/pypy/interpreter/pyparser/automata.py b/pypy/interpreter/pyparser/automata.py --- a/pypy/interpreter/pyparser/automata.py +++ b/pypy/interpreter/pyparser/automata.py @@ -23,6 +23,10 @@ ERROR_STATE = chr(255) +# NB: all non-ascii bytes (>= 128) will be turned into 128 +NON_ASCII = chr(128) + + class DFA: # ____________________________________________________________ def __init__(self, states, accepts, start = 0): @@ -36,7 +40,10 @@ for key in state: if key == DEFAULT: continue - maximum = max(ord(key), maximum) + ordkey = ord(key) + if ordkey > 128: + raise ValueError("DFA does not support matching of specific non-ASCII character %r. Use NON_ASCII instead" % key) + maximum = max(ordkey, maximum) self.max_char = maximum + 1 defaults = [] @@ -72,6 +79,8 @@ i = pos for i in range(pos, len(inVec)): item = inVec[i] + if ord(item) > 0x80: + item = NON_ASCII accept = self.accepts[crntState] crntState = self._next_state(item, crntState) if crntState != ERROR_STATE: @@ -103,6 +112,8 @@ i = pos for i in range(pos, len(inVec)): item = inVec[i] + if ord(item) > 0x80: + item = NON_ASCII accept = self.accepts[crntState] if accept: return i diff --git a/pypy/interpreter/pyparser/data/Grammar2.7 b/pypy/interpreter/pyparser/data/Grammar2.7 --- a/pypy/interpreter/pyparser/data/Grammar2.7 +++ b/pypy/interpreter/pyparser/data/Grammar2.7 @@ -104,7 +104,7 @@ '[' [listmaker] ']' | '{' [dictorsetmaker] '}' | '`' testlist1 '`' | - NAME | NUMBER | STRING+) + NAME | NUMBER | STRING+ | '$NUM') listmaker: test ( list_for | (',' test)* [','] ) testlist_comp: test ( comp_for | (',' test)* [','] ) lambdef: 'lambda' [varargslist] ':' test diff --git a/pypy/interpreter/pyparser/dfa_generated.py b/pypy/interpreter/pyparser/dfa_generated.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/dfa_generated.py @@ -0,0 +1,299 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY gendfa.py +# DO NOT EDIT +# TO REGENERATE THE FILE, RUN: +# python gendfa.py > dfa_generated.py + +from pypy.interpreter.pyparser import automata +accepts = [True, True, True, True, True, True, True, True, + True, True, False, True, True, True, True, False, + False, False, False, True, False, False, True, + False, False, True, False, True, True, False, + True, False, False, True, False, False, True, + True, True, False, False, True, False, False, + False, True] +states = [ + # 0 + {'\t': 0, '\n': 13, '\x0c': 0, + '\r': 14, ' ': 0, '!': 10, '"': 17, + '#': 19, '$': 15, '%': 12, '&': 12, + "'": 16, '(': 13, ')': 13, '*': 7, + '+': 12, ',': 13, '-': 12, '.': 6, + '/': 11, '0': 4, '1': 5, '2': 5, + '3': 5, '4': 5, '5': 5, '6': 5, + '7': 5, '8': 5, '9': 5, ':': 13, + ';': 13, '<': 9, '=': 12, '>': 8, + '@': 13, 'A': 1, 'B': 2, 'C': 1, + 'D': 1, 'E': 1, 'F': 1, 'G': 1, + 'H': 1, 'I': 1, 'J': 1, 'K': 1, + 'L': 1, 'M': 1, 'N': 1, 'O': 1, + 'P': 1, 'Q': 1, 'R': 3, 'S': 1, + 'T': 1, 'U': 2, 'V': 1, 'W': 1, + 'X': 1, 'Y': 1, 'Z': 1, '[': 13, + '\\': 18, ']': 13, '^': 12, '_': 1, + '`': 13, 'a': 1, 'b': 2, 'c': 1, + 'd': 1, 'e': 1, 'f': 1, 'g': 1, + 'h': 1, 'i': 1, 'j': 1, 'k': 1, + 'l': 1, 'm': 1, 'n': 1, 'o': 1, + 'p': 1, 'q': 1, 'r': 3, 's': 1, + 't': 1, 'u': 2, 'v': 1, 'w': 1, + 'x': 1, 'y': 1, 'z': 1, '{': 13, + '|': 12, '}': 13, '~': 13}, + # 1 + {'0': 1, '1': 1, '2': 1, '3': 1, + '4': 1, '5': 1, '6': 1, '7': 1, + '8': 1, '9': 1, 'A': 1, 'B': 1, + 'C': 1, 'D': 1, 'E': 1, 'F': 1, + 'G': 1, 'H': 1, 'I': 1, 'J': 1, + 'K': 1, 'L': 1, 'M': 1, 'N': 1, + 'O': 1, 'P': 1, 'Q': 1, 'R': 1, + 'S': 1, 'T': 1, 'U': 1, 'V': 1, + 'W': 1, 'X': 1, 'Y': 1, 'Z': 1, + '_': 1, 'a': 1, 'b': 1, 'c': 1, + 'd': 1, 'e': 1, 'f': 1, 'g': 1, + 'h': 1, 'i': 1, 'j': 1, 'k': 1, + 'l': 1, 'm': 1, 'n': 1, 'o': 1, + 'p': 1, 'q': 1, 'r': 1, 's': 1, + 't': 1, 'u': 1, 'v': 1, 'w': 1, + 'x': 1, 'y': 1, 'z': 1}, + # 2 + {'"': 17, "'": 16, '0': 1, '1': 1, + '2': 1, '3': 1, '4': 1, '5': 1, + '6': 1, '7': 1, '8': 1, '9': 1, + 'A': 1, 'B': 1, 'C': 1, 'D': 1, + 'E': 1, 'F': 1, 'G': 1, 'H': 1, + 'I': 1, 'J': 1, 'K': 1, 'L': 1, + 'M': 1, 'N': 1, 'O': 1, 'P': 1, + 'Q': 1, 'R': 3, 'S': 1, 'T': 1, + 'U': 1, 'V': 1, 'W': 1, 'X': 1, + 'Y': 1, 'Z': 1, '_': 1, 'a': 1, + 'b': 1, 'c': 1, 'd': 1, 'e': 1, + 'f': 1, 'g': 1, 'h': 1, 'i': 1, + 'j': 1, 'k': 1, 'l': 1, 'm': 1, + 'n': 1, 'o': 1, 'p': 1, 'q': 1, + 'r': 3, 's': 1, 't': 1, 'u': 1, + 'v': 1, 'w': 1, 'x': 1, 'y': 1, + 'z': 1}, + # 3 + {'"': 17, "'": 16, '0': 1, '1': 1, + '2': 1, '3': 1, '4': 1, '5': 1, + '6': 1, '7': 1, '8': 1, '9': 1, + 'A': 1, 'B': 1, 'C': 1, 'D': 1, + 'E': 1, 'F': 1, 'G': 1, 'H': 1, + 'I': 1, 'J': 1, 'K': 1, 'L': 1, + 'M': 1, 'N': 1, 'O': 1, 'P': 1, + 'Q': 1, 'R': 1, 'S': 1, 'T': 1, + 'U': 1, 'V': 1, 'W': 1, 'X': 1, + 'Y': 1, 'Z': 1, '_': 1, 'a': 1, + 'b': 1, 'c': 1, 'd': 1, 'e': 1, + 'f': 1, 'g': 1, 'h': 1, 'i': 1, + 'j': 1, 'k': 1, 'l': 1, 'm': 1, + 'n': 1, 'o': 1, 'p': 1, 'q': 1, + 'r': 1, 's': 1, 't': 1, 'u': 1, + 'v': 1, 'w': 1, 'x': 1, 'y': 1, + 'z': 1}, + # 4 + {'.': 25, '0': 22, '1': 22, '2': 22, + '3': 22, '4': 22, '5': 22, '6': 22, + '7': 22, '8': 24, '9': 24, 'B': 23, + 'E': 26, 'J': 13, 'L': 13, 'O': 21, + 'X': 20, 'b': 23, 'e': 26, 'j': 13, + 'l': 13, 'o': 21, 'x': 20}, + # 5 + {'.': 25, '0': 5, '1': 5, '2': 5, + '3': 5, '4': 5, '5': 5, '6': 5, + '7': 5, '8': 5, '9': 5, 'E': 26, + 'J': 13, 'L': 13, 'e': 26, 'j': 13, + 'l': 13}, + # 6 + {'0': 27, '1': 27, '2': 27, '3': 27, + '4': 27, '5': 27, '6': 27, '7': 27, + '8': 27, '9': 27}, + # 7 + {'*': 12, '=': 13}, + # 8 + {'=': 13, '>': 12}, + # 9 + {'<': 12, '=': 13, '>': 13}, + # 10 + {'=': 13}, + # 11 + {'/': 12, '=': 13}, + # 12 + {'=': 13}, + # 13 + {}, + # 14 + {'\n': 13}, + # 15 + {'0': 28, '1': 28, '2': 28, '3': 28, + '4': 28, '5': 28, '6': 28, '7': 28, + '8': 28, '9': 28}, + # 16 + {automata.DEFAULT: 32, '\n': 29, + '\r': 29, "'": 30, '\\': 31}, + # 17 + {automata.DEFAULT: 35, '\n': 29, + '\r': 29, '"': 33, '\\': 34}, + # 18 + {'\n': 13, '\r': 14}, + # 19 + {automata.DEFAULT: 19, '\n': 29, '\r': 29}, + # 20 + {'0': 36, '1': 36, '2': 36, '3': 36, + '4': 36, '5': 36, '6': 36, '7': 36, + '8': 36, '9': 36, 'A': 36, 'B': 36, + 'C': 36, 'D': 36, 'E': 36, 'F': 36, + 'a': 36, 'b': 36, 'c': 36, 'd': 36, + 'e': 36, 'f': 36}, + # 21 + {'0': 37, '1': 37, '2': 37, '3': 37, + '4': 37, '5': 37, '6': 37, '7': 37}, + # 22 + {'.': 25, '0': 22, '1': 22, '2': 22, + '3': 22, '4': 22, '5': 22, '6': 22, + '7': 22, '8': 24, '9': 24, 'E': 26, + 'J': 13, 'L': 13, 'e': 26, 'j': 13, + 'l': 13}, + # 23 + {'0': 38, '1': 38}, + # 24 + {'.': 25, '0': 24, '1': 24, '2': 24, + '3': 24, '4': 24, '5': 24, '6': 24, + '7': 24, '8': 24, '9': 24, 'E': 26, + 'J': 13, 'e': 26, 'j': 13}, + # 25 + {'0': 25, '1': 25, '2': 25, '3': 25, + '4': 25, '5': 25, '6': 25, '7': 25, + '8': 25, '9': 25, 'E': 39, 'J': 13, + 'e': 39, 'j': 13}, + # 26 + {'+': 40, '-': 40, '0': 41, '1': 41, + '2': 41, '3': 41, '4': 41, '5': 41, + '6': 41, '7': 41, '8': 41, '9': 41}, + # 27 + {'0': 27, '1': 27, '2': 27, '3': 27, + '4': 27, '5': 27, '6': 27, '7': 27, + '8': 27, '9': 27, 'E': 39, 'J': 13, + 'e': 39, 'j': 13}, + # 28 + {'0': 28, '1': 28, '2': 28, '3': 28, + '4': 28, '5': 28, '6': 28, '7': 28, + '8': 28, '9': 28}, + # 29 + {}, + # 30 + {"'": 13}, + # 31 + {automata.DEFAULT: 42, '\n': 13, '\r': 14}, + # 32 + {automata.DEFAULT: 32, '\n': 29, + '\r': 29, "'": 13, '\\': 31}, + # 33 + {'"': 13}, + # 34 + {automata.DEFAULT: 43, '\n': 13, '\r': 14}, + # 35 + {automata.DEFAULT: 35, '\n': 29, + '\r': 29, '"': 13, '\\': 34}, + # 36 + {'0': 36, '1': 36, '2': 36, '3': 36, + '4': 36, '5': 36, '6': 36, '7': 36, + '8': 36, '9': 36, 'A': 36, 'B': 36, + 'C': 36, 'D': 36, 'E': 36, 'F': 36, + 'L': 13, 'a': 36, 'b': 36, 'c': 36, + 'd': 36, 'e': 36, 'f': 36, 'l': 13}, + # 37 + {'0': 37, '1': 37, '2': 37, '3': 37, + '4': 37, '5': 37, '6': 37, '7': 37, + 'L': 13, 'l': 13}, + # 38 + {'0': 38, '1': 38, 'L': 13, 'l': 13}, + # 39 + {'+': 44, '-': 44, '0': 45, '1': 45, + '2': 45, '3': 45, '4': 45, '5': 45, + '6': 45, '7': 45, '8': 45, '9': 45}, + # 40 + {'0': 41, '1': 41, '2': 41, '3': 41, + '4': 41, '5': 41, '6': 41, '7': 41, + '8': 41, '9': 41}, + # 41 + {'0': 41, '1': 41, '2': 41, '3': 41, + '4': 41, '5': 41, '6': 41, '7': 41, + '8': 41, '9': 41, 'J': 13, 'j': 13}, + # 42 + {automata.DEFAULT: 42, '\n': 29, + '\r': 29, "'": 13, '\\': 31}, + # 43 + {automata.DEFAULT: 43, '\n': 29, + '\r': 29, '"': 13, '\\': 34}, + # 44 + {'0': 45, '1': 45, '2': 45, '3': 45, + '4': 45, '5': 45, '6': 45, '7': 45, + '8': 45, '9': 45}, + # 45 + {'0': 45, '1': 45, '2': 45, '3': 45, + '4': 45, '5': 45, '6': 45, '7': 45, + '8': 45, '9': 45, 'J': 13, 'j': 13}, + ] +pseudoDFA = automata.DFA(states, accepts) + +accepts = [False, False, False, False, False, True] +states = [ + # 0 + {automata.DEFAULT: 0, '"': 1, '\\': 2}, + # 1 + {automata.DEFAULT: 4, '"': 3, '\\': 2}, + # 2 + {automata.DEFAULT: 4}, + # 3 + {automata.DEFAULT: 4, '"': 5, '\\': 2}, + # 4 + {automata.DEFAULT: 4, '"': 1, '\\': 2}, + # 5 + {automata.DEFAULT: 4, '"': 5, '\\': 2}, + ] +double3DFA = automata.NonGreedyDFA(states, accepts) + +accepts = [False, False, False, False, False, True] +states = [ + # 0 + {automata.DEFAULT: 0, "'": 1, '\\': 2}, + # 1 + {automata.DEFAULT: 4, "'": 3, '\\': 2}, + # 2 + {automata.DEFAULT: 4}, + # 3 + {automata.DEFAULT: 4, "'": 5, '\\': 2}, + # 4 + {automata.DEFAULT: 4, "'": 1, '\\': 2}, + # 5 + {automata.DEFAULT: 4, "'": 5, '\\': 2}, + ] +single3DFA = automata.NonGreedyDFA(states, accepts) + +accepts = [False, True, False, False] +states = [ + # 0 + {automata.DEFAULT: 0, "'": 1, '\\': 2}, + # 1 + {}, + # 2 + {automata.DEFAULT: 3}, + # 3 + {automata.DEFAULT: 3, "'": 1, '\\': 2}, + ] +singleDFA = automata.DFA(states, accepts) + +accepts = [False, True, False, False] +states = [ + # 0 + {automata.DEFAULT: 0, '"': 1, '\\': 2}, + # 1 + {}, + # 2 + {automata.DEFAULT: 3}, + # 3 + {automata.DEFAULT: 3, '"': 1, '\\': 2}, + ] +doubleDFA = automata.DFA(states, accepts) + diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py --- a/pypy/interpreter/pyparser/future.py +++ b/pypy/interpreter/pyparser/future.py @@ -43,7 +43,7 @@ self.tok = self.tokens[index] def skip(self, n): - if self.tok[0] == n: + if self.tok.token_type == n: self.next() return True else: @@ -51,7 +51,7 @@ def skip_name(self, name): from pypy.interpreter.pyparser import pygram - if self.tok[0] == pygram.tokens.NAME and self.tok[1] == name: + if self.tok.token_type == pygram.tokens.NAME and self.tok.value == name: self.next() return True else: @@ -59,8 +59,8 @@ def next_feature_name(self): from pypy.interpreter.pyparser import pygram - if self.tok[0] == pygram.tokens.NAME: - name = self.tok[1] + if self.tok.token_type == pygram.tokens.NAME: + name = self.tok.value self.next() if self.skip_name("as"): self.skip(pygram.tokens.NAME) @@ -101,7 +101,7 @@ # somewhere inside the last __future__ import statement # (at the start would be fine too, but it's easier to grab a # random position inside) - last_position = (it.tok[2], it.tok[3]) + last_position = (it.tok.lineno, it.tok.column) result |= future_flags.get_compiler_feature(it.next_feature_name()) while it.skip(pygram.tokens.COMMA): result |= future_flags.get_compiler_feature(it.next_feature_name()) diff --git a/pypy/interpreter/pyparser/gendfa.py b/pypy/interpreter/pyparser/gendfa.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/gendfa.py @@ -0,0 +1,353 @@ +#! /usr/bin/env python +"""Module gendfa + +Generates finite state automata for recognizing Python tokens. These are hand +coded versions of the regular expressions originally appearing in Ping's +tokenize module in the Python standard library. + +When run from the command line, this should pretty print the DFA machinery. + +To regenerate the dfa, run:: + + $ python gendfa.py > dfa_generated.py + +$Id: genPytokenize.py,v 1.1 2003/10/02 17:37:17 jriehl Exp $ +""" + +from pypy.interpreter.pyparser.pylexer import * +from pypy.interpreter.pyparser.automata import NonGreedyDFA, DFA, DEFAULT + +def makePyPseudoDFA (): + import string + states = [] + def makeEOL(): + return group(states, + newArcPair(states, "\n"), + chain(states, + newArcPair(states, "\r"), + maybe(states, newArcPair(states, "\n")))) + # ____________________________________________________________ + def makeLineCont (): + return chain(states, + newArcPair(states, "\\"), + makeEOL()) + # ____________________________________________________________ + # Ignore stuff + def makeWhitespace (): + return any(states, groupStr(states, " \f\t")) + # ____________________________________________________________ + def makeComment (): + return chain(states, + newArcPair(states, "#"), + any(states, notGroupStr(states, "\r\n"))) + # ____________________________________________________________ + #ignore = chain(states, + # makeWhitespace(), + # any(states, chain(states, + # makeLineCont(), + # makeWhitespace())), + # maybe(states, makeComment())) + # ____________________________________________________________ + # Names + name = chain(states, + groupStr(states, string.letters + "_"), + any(states, groupStr(states, + string.letters + string.digits + "_"))) + # ____________________________________________________________ + # Digits + def makeDigits (): From pypy.commits at gmail.com Thu Oct 11 05:23:54 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 11 Oct 2018 02:23:54 -0700 (PDT) Subject: [pypy-commit] cffi default: #389 Message-ID: <5bbf16aa.1c69fb81.20966.4ecb@mx.google.com> Author: Armin Rigo Branch: Changeset: r3168:f2522654c2b5 Date: 2018-10-11 11:23 +0200 http://bitbucket.org/cffi/cffi/changeset/f2522654c2b5/ Log: #389 Warn when using string literals in the cdef() source diff --git a/cffi/cparser.py b/cffi/cparser.py --- a/cffi/cparser.py +++ b/cffi/cparser.py @@ -137,6 +137,14 @@ parts.append(csource) return ''.join(parts) +def _warn_for_string_literal(csource): + if '"' in csource: + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + def _preprocess(csource): # Remove comments. NOTE: this only work because the cdef() section # should not contain any string literal! @@ -148,6 +156,7 @@ macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) + _warn_for_string_literal(csource) # if pycparser.__version__ < '2.14': csource = _workaround_for_old_pycparser(csource) From pypy.commits at gmail.com Thu Oct 11 17:26:27 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 11 Oct 2018 14:26:27 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: add, test LoadLibraryW, LoadLibraryEx{A, W} Message-ID: <5bbfc003.1c69fb81.a7a8e.691d@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r95201:ac3e7351541c Date: 2018-10-07 12:10 +0300 http://bitbucket.org/pypy/pypy/changeset/ac3e7351541c/ Log: add, test LoadLibraryW, LoadLibraryEx{A, W} diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -113,6 +113,7 @@ MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR + LOAD_WITH_ALTERED_SEARCH_PATH """ from rpython.translator.platform import host_factory static_platform = host_factory() @@ -195,6 +196,22 @@ GetModuleHandle = winexternal('GetModuleHandleA', [rffi.CCHARP], HMODULE) LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) + def wrap_loadlibraryex(func): + def loadlibrary(name, handle=None, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + # Requires a full path name with '/' -> '\\' + return func(name, handle, flags) + return loadlibrary + + _LoadLibraryExA = winexternal('LoadLibraryExA', + [rffi.CCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExA = wrap_loadlibraryex(_LoadLibraryExA) + LoadLibraryW = winexternal('LoadLibraryW', [rffi.CWCHARP], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + _LoadLibraryExW = winexternal('LoadLibraryExW', + [rffi.CWCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExW = wrap_loadlibraryex(_LoadLibraryExW) GetProcAddress = winexternal('GetProcAddress', [HMODULE, rffi.CCHARP], rffi.VOIDP) diff --git a/rpython/rlib/test/loadtest/loadtest0.dll b/rpython/rlib/test/loadtest/loadtest0.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9bdcc33a1902f8e989d349c49c2cc08e633aa32b GIT binary patch [cut] diff --git a/rpython/rlib/test/loadtest/loadtest1.dll b/rpython/rlib/test/loadtest/loadtest1.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cb83854875c876717371bdf90488eed9c6571f03 GIT binary patch [cut] diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -6,6 +6,44 @@ from rpython.rlib import rwin32 from rpython.tool.udir import udir +loadtest_dir = os.path.dirname(__file__) + '/loadtest' +test1 = os.path.abspath(loadtest_dir + '/loadtest1.dll') +test0 = os.path.abspath(loadtest_dir + '/loadtest0.dll') + +if not os.path.exists(test1) or not os.path.exists(test0): + # This is how the files, which are checked into the repo, were created + from rpython.translator.tool.cbuild import ExternalCompilationInfo + from rpython.translator.platform import platform + from rpython.translator import cdir + if not os.path.exists(loadtest_dir): + os.mkdir(loadtest_dir) + c_file = udir.ensure("test_rwin32", dir=1).join("test0.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + RPY_EXPORTED + int internal_sum(int a, int b) { + return a + b; + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir]) + lib_name = str(platform.compile([c_file], eci, test0[:-4], + standalone=False)) + assert os.path.abspath(lib_name) == os.path.abspath(test0) + + c_file = udir.ensure("test_rwin32", dir=1).join("test1.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + int internal_sum(int a, int b); + RPY_EXPORTED + int sum(int a, int b) { + return internal_sum(a, b); + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir], + libraries=[loadtest_dir + '/loadtest0']) + lib_name = str(platform.compile([c_file], eci, test1[:-4], + standalone=False, )) + assert os.path.abspath(lib_name) == os.path.abspath(test1) def test_get_osfhandle(): fid = open(str(udir.join('validate_test.txt')), 'w') @@ -28,13 +66,13 @@ "import time;" "time.sleep(10)", ], - ) + ) print proc.pid handle = rwin32.OpenProcess(rwin32.PROCESS_ALL_ACCESS, False, proc.pid) assert rwin32.TerminateProcess(handle, signal.SIGTERM) == 1 rwin32.CloseHandle(handle) assert proc.wait() == signal.SIGTERM - + @py.test.mark.dont_track_allocations('putenv intentionally keeps strings alive') def test_wenviron(): name, value = u'PYPY_TEST_日本', u'foobar日本' @@ -55,3 +93,48 @@ msg = rwin32.FormatErrorW(34) assert type(msg) is unicode assert u'%2' in msg + +def test_loadlibraryA(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibrary(test0) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibrary(test1) + assert not hdll + + assert os.path.exists(test1) + + hdll = rwin32.LoadLibraryExA(test1) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + +def test_loadlibraryW(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibraryW(unicode(test0)) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibraryW(unicode(test1)) + assert not hdll + + assert os.path.exists(unicode(test1)) + + hdll = rwin32.LoadLibraryExW(unicode(test1)) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + +def test_loadlibrary_unicode(): + import shutil + test0u = unicode(udir.join(u'load\u03betest.dll')) + shutil.copyfile(test0, test0u) + hdll = rwin32.LoadLibraryW(test0u) + assert hdll From pypy.commits at gmail.com Thu Oct 11 17:26:29 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 11 Oct 2018 14:26:29 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: Use LoadLibraryEx in loading extension modules, like CPython's _PyImport_FindSharedFuncptr Message-ID: <5bbfc005.1c69fb81.ed7cf.4a6c@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r95202:2dda0ffebf61 Date: 2018-10-07 14:20 +0300 http://bitbucket.org/pypy/pypy/changeset/2dda0ffebf61/ Log: Use LoadLibraryEx in loading extension modules, like CPython's _PyImport_FindSharedFuncptr diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -93,8 +93,11 @@ if sys.platform == 'win32': dash = '_' + WIN32 = True else: dash = '' + WIN32 = False + def fclose(fp): try: @@ -1671,7 +1674,11 @@ try: ll_libname = rffi.str2charp(path) try: - dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) + if WIN32: + # Allow other DLLs in the same directory with "path" + dll = rdynload.dlopenex(ll_libname) + else: + dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -233,6 +233,15 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenex(name): + res = rwin32.LoadLibraryExA(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: From pypy.commits at gmail.com Thu Oct 11 17:26:31 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 11 Oct 2018 14:26:31 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: fix win32 translation, also remove some MSVC compiler warnings Message-ID: <5bbfc007.1c69fb81.4f6cf.7e82@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r95203:f59b3a27e35f Date: 2018-10-09 17:23 +0300 http://bitbucket.org/pypy/pypy/changeset/f59b3a27e35f/ Log: fix win32 translation, also remove some MSVC compiler warnings diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -95,7 +95,9 @@ if os.name == 'nt': do_includes = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include static void _cffi_init(void); diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1564,6 +1564,9 @@ } ''' separate_module_sources.append(get_pythonapi_source) + kwds['post_include_bits'] = [ + 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', + ] eci = ExternalCompilationInfo( include_dirs=include_dirs, diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -231,7 +231,9 @@ if os.name == 'nt': _source_code = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include #include diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -835,7 +835,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = rwin32.get_osfhandle(fileno) + fh = rffi.cast(HANDLE, rwin32.get_osfhandle(fileno)) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,7 +137,10 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', + 'RPY_EXTERN void* enter_suppress_iph();', + 'RPY_EXTERN void exit_suppress_iph(void* handle);', + ] else: separate_module_sources = [] post_include_bits = [] @@ -235,7 +238,8 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', + 'direct.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -20,7 +20,7 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h', 'stdio.h', 'stdlib.h'], + includes = ['windows.h', 'stdio.h', 'stdlib.h', 'io.h'], libraries = ['kernel32'], ) else: @@ -197,9 +197,9 @@ LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) def wrap_loadlibraryex(func): - def loadlibrary(name, handle=None, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + def loadlibrary(name, flags=LOAD_WITH_ALTERED_SEARCH_PATH): # Requires a full path name with '/' -> '\\' - return func(name, handle, flags) + return func(name, NULL_HANDLE, flags) return loadlibrary _LoadLibraryExA = winexternal('LoadLibraryExA', @@ -217,7 +217,7 @@ rffi.VOIDP) FreeLibrary = winexternal('FreeLibrary', [HMODULE], BOOL, releasegil=False) - LocalFree = winexternal('LocalFree', [HLOCAL], DWORD) + LocalFree = winexternal('LocalFree', [HLOCAL], HLOCAL) CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL, releasegil=False, save_err=rffi.RFFI_SAVE_LASTERROR) CloseHandle_no_err = winexternal('CloseHandle', [HANDLE], BOOL, @@ -232,12 +232,12 @@ [DWORD, rffi.VOIDP, DWORD, DWORD, rffi.CWCHARP, DWORD, rffi.VOIDP], DWORD) - _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) + _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], rffi.INTP) def get_osfhandle(fd): from rpython.rlib.rposix import FdValidator with FdValidator(fd): - handle = _get_osfhandle(fd) + handle = rffi.cast(HANDLE, _get_osfhandle(fd)) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle From pypy.commits at gmail.com Thu Oct 11 17:27:50 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 11 Oct 2018 14:27:50 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: move include before declaration Message-ID: <5bbfc056.1c69fb81.bdf73.95c0@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r95204:dfef27e92951 Date: 2018-10-10 12:06 +0300 http://bitbucket.org/pypy/pypy/changeset/dfef27e92951/ Log: move include before declaration diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1550,7 +1550,6 @@ if sys.platform == 'win32': get_pythonapi_source = ''' - #include RPY_EXTERN HANDLE pypy_get_pythonapi_handle() { MEMORY_BASIC_INFORMATION mi; @@ -1564,7 +1563,7 @@ } ''' separate_module_sources.append(get_pythonapi_source) - kwds['post_include_bits'] = [ + kwds['post_include_bits'] = ['#include ', 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', ] From pypy.commits at gmail.com Fri Oct 12 03:36:36 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 12 Oct 2018 00:36:36 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: update nt scheme from cpython Message-ID: <5bc04f04.1c69fb81.71adb.2f86@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95206:1c4130fcccfa Date: 2018-10-12 10:35 +0300 http://bitbucket.org/pypy/pypy/changeset/1c4130fcccfa/ Log: update nt scheme from cpython diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py --- a/lib-python/3/distutils/sysconfig_pypy.py +++ b/lib-python/3/distutils/sysconfig_pypy.py @@ -8,11 +8,9 @@ available. """ -__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $" - import sys import os -import imp +import imp, _imp from distutils.errors import DistutilsPlatformError @@ -71,9 +69,17 @@ def _init_nt(): """Initialize the module as appropriate for NT""" g = {} + # set basic install directories + g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) + g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) + + # XXX hmmm.. a normal install puts include files here + g['INCLUDEPY'] = get_python_inc(plat_specific=0) + + g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] g['EXE'] = ".exe" - g['SO'] = ".pyd" - g['SOABI'] = g['SO'].rsplit('.')[0] # xxx? + g['VERSION'] = get_python_version().replace(".", "") + g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) global _config_vars _config_vars = g From pypy.commits at gmail.com Fri Oct 12 03:36:39 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 12 Oct 2018 00:36:39 -0700 (PDT) Subject: [pypy-commit] pypy release-pypy3.5-6.x: update nt scheme from cpython Message-ID: <5bc04f07.1c69fb81.ddf75.3998@mx.google.com> Author: Matti Picus Branch: release-pypy3.5-6.x Changeset: r95207:dc0bff39e4bb Date: 2018-10-12 10:35 +0300 http://bitbucket.org/pypy/pypy/changeset/dc0bff39e4bb/ Log: update nt scheme from cpython diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py --- a/lib-python/3/distutils/sysconfig_pypy.py +++ b/lib-python/3/distutils/sysconfig_pypy.py @@ -8,11 +8,9 @@ available. """ -__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $" - import sys import os -import imp +import imp, _imp from distutils.errors import DistutilsPlatformError @@ -101,9 +99,17 @@ def _init_nt(): """Initialize the module as appropriate for NT""" g = {} + # set basic install directories + g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) + g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) + + # XXX hmmm.. a normal install puts include files here + g['INCLUDEPY'] = get_python_inc(plat_specific=0) + + g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] g['EXE'] = ".exe" - g['SO'] = ".pyd" - g['SOABI'] = g['SO'].rsplit('.')[0] # xxx? + g['VERSION'] = get_python_version().replace(".", "") + g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) global _config_vars _config_vars = g From pypy.commits at gmail.com Fri Oct 12 08:40:17 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 12 Oct 2018 05:40:17 -0700 (PDT) Subject: [pypy-commit] pypy default: record test durations - why is win32 so slow? Message-ID: <5bc09631.1c69fb81.db895.47ab@mx.google.com> Author: Matti Picus Branch: Changeset: r95208:92acfaf3e806 Date: 2018-10-12 15:38 +0300 http://bitbucket.org/pypy/pypy/changeset/92acfaf3e806/ Log: record test durations - why is win32 so slow? diff --git a/testrunner/lib_python_tests.py b/testrunner/lib_python_tests.py --- a/testrunner/lib_python_tests.py +++ b/testrunner/lib_python_tests.py @@ -15,6 +15,7 @@ [sys.executable, "pypy/test_all.py", "--pypy=pypy/goal/pypy-c", "--timeout=3600", + "--duration=10", "--resultlog=cpython.log", "lib-python", ] + sys.argv[1:], cwd=rootdir) From pypy.commits at gmail.com Fri Oct 12 08:40:20 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 12 Oct 2018 05:40:20 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into branch Message-ID: <5bc09634.1c69fb81.f2b89.7916@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95209:16a9478214da Date: 2018-10-12 15:39 +0300 http://bitbucket.org/pypy/pypy/changeset/16a9478214da/ Log: merge default into branch diff --git a/testrunner/lib_python_tests.py b/testrunner/lib_python_tests.py --- a/testrunner/lib_python_tests.py +++ b/testrunner/lib_python_tests.py @@ -21,6 +21,7 @@ pypyopt, "--timeout=3600", "-rs", + "--duration=10", "--resultlog=cpython.log", "lib-python", ] + sys.argv[1:], cwd=rootdir) From pypy.commits at gmail.com Fri Oct 12 09:03:09 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 12 Oct 2018 06:03:09 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5bc09b8d.1c69fb81.d7296.90bd@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95210:07afb9936e34 Date: 2018-10-12 15:56 +0300 http://bitbucket.org/pypy/pypy/changeset/07afb9936e34/ Log: merge py3.5 into branch diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py --- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py +++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py @@ -186,7 +186,7 @@ # Prefer poll, if available, since you can poll() any fd # which can't be done with select(). if HAVE_POLL: - p.register(sock.fileno(), POLLOUT | POLLIN) + p.register(sock.fileno(), POLLOUT if writing else POLLIN) rc = len(p.poll(timeout * 1000.0)) else: diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -79,13 +79,6 @@ w_result = self.ctype.cast_to_int(ptr) return w_result - def long(self, space): - w_result = self.int(space) - space = self.space - if space.is_w(space.type(w_result), space.w_int): - w_result = space.newlong(space.int_w(w_result)) - return w_result - def float(self): with self as ptr: w_result = self.ctype.float(ptr) @@ -664,7 +657,6 @@ __repr__ = interp2app(W_CData.repr), __bool__ = interp2app(W_CData.bool), __int__ = interp2app(W_CData.int), - __long__ = interp2app(W_CData.long), __float__ = interp2app(W_CData.float), __complex__ = interp2app(W_CData.complex), __len__ = interp2app(W_CData.len), diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -95,7 +95,9 @@ if os.name == 'nt': do_includes = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include static void _cffi_init(void); diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -6,6 +6,7 @@ from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.nonconst import NonConstant from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -202,6 +203,9 @@ else: if strict and value < 0: raise OperationError(space.w_OverflowError, space.newtext(neg_msg)) + if not we_are_translated(): + if isinstance(value, NonConstant): # hack for test_ztranslation + return r_uint(0) return r_uint(value) # note that if not 'strict', then space.int() will round down floats bigint = space.bigint_w(space.int(w_ob), allow_conversion=False) diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -772,6 +772,26 @@ assert b"\\u3042\u3xxx".decode("unicode-escape", "test.handler1") == \ u"\u3042[<92><117><51>]xxx" + def test_unicode_internal_error_handler_infinite_loop(self): + import codecs + class MyException(Exception): + pass + seen = [0] + def handler_unicodeinternal(exc): + if not isinstance(exc, UnicodeDecodeError): + raise TypeError("don't know how to handle %r" % exc) + seen[0] += 1 + if seen[0] == 20: # stop the 20th time this is called + raise MyException + return (u"\x01", 4) # 4 < len(input), so will try and fail again + codecs.register_error("test.inf", handler_unicodeinternal) + try: + b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.inf") + except MyException: + pass + else: + raise AssertionError("should have gone into infinite loop") + def test_encode_error_bad_handler(self): import codecs codecs.register_error("test.bad_handler", lambda e: (repl, 1)) diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -196,12 +196,11 @@ wrap_oserror2(space, e, w_name, exception_name='w_IOError', eintr_retry=True) - if not rposix._WIN32: - try: - _open_inhcache.set_non_inheritable(self.fd) - except OSError as e: - raise wrap_oserror2(space, e, w_name, - eintr_retry=False) + try: + _open_inhcache.set_non_inheritable(self.fd) + except OSError as e: + raise wrap_oserror2(space, e, w_name, + eintr_retry=False) else: w_fd = space.call_function(w_opener, w_name, space.newint(flags)) @@ -225,6 +224,7 @@ raise wrap_oserror2(space, e, w_name, eintr_retry=False) + try: st = os.fstat(self.fd) except OSError as e: diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -279,31 +279,34 @@ raises(FileExistsError, _io.FileIO, filename, 'x') def test_non_inheritable(self): - import _io, posix + import _io + os = self.posix f = _io.FileIO(self.tmpfile, 'r') - assert posix.get_inheritable(f.fileno()) == False + assert os.get_inheritable(f.fileno()) == False f.close() def test_FileIO_fd_does_not_change_inheritable(self): - import _io, posix - fd1, fd2 = posix.pipe() - posix.set_inheritable(fd1, True) - posix.set_inheritable(fd2, False) + import _io + os = self.posix + fd1, fd2 = os.pipe() + os.set_inheritable(fd1, True) + os.set_inheritable(fd2, False) f1 = _io.FileIO(fd1, 'r') f2 = _io.FileIO(fd2, 'w') - assert posix.get_inheritable(fd1) == True - assert posix.get_inheritable(fd2) == False + assert os.get_inheritable(fd1) == True + assert os.get_inheritable(fd2) == False f1.close() f2.close() def test_close_upon_reinit(self): - import _io, posix + import _io + os = self.posix f = _io.FileIO(self.tmpfile, 'r') fd1 = f.fileno() f.__init__(self.tmpfile, 'w') fd2 = f.fileno() if fd1 != fd2: - raises(OSError, posix.close, fd1) + raises(OSError, os.close, fd1) def test_opener_negative(self): import _io diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -93,8 +93,11 @@ if sys.platform == 'win32': dash = '_' + WIN32 = True else: dash = '' + WIN32 = False + def fclose(fp): try: @@ -610,13 +613,9 @@ 'PyObject_CallMethod', 'PyObject_CallFunctionObjArgs', 'PyObject_CallMethodObjArgs', '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT', - 'PyObject_GetBuffer', 'PyBuffer_Release', + 'PyObject_DelItemString', 'PyObject_GetBuffer', 'PyBuffer_Release', '_Py_setfilesystemdefaultencoding', - 'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr', - 'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr', - 'PyCObject_Type', '_Py_get_cobject_type', - 'PyCapsule_New', 'PyCapsule_IsValid', 'PyCapsule_GetPointer', 'PyCapsule_GetName', 'PyCapsule_GetDestructor', 'PyCapsule_GetContext', 'PyCapsule_SetPointer', 'PyCapsule_SetName', 'PyCapsule_SetDestructor', @@ -672,7 +671,7 @@ 'PyObject*', 'space.w_None', header=pypy_decl) register_global('_Py_TrueStruct', 'PyObject*', 'space.w_True', header=pypy_decl) -register_global('_Py_ZeroStruct', +register_global('_Py_FalseStruct', 'PyObject*', 'space.w_False', header=pypy_decl) register_global('_Py_NotImplementedStruct', 'PyObject*', 'space.w_NotImplemented', header=pypy_decl) @@ -716,8 +715,8 @@ "PyByteArray_Type": "space.w_bytearray", "PyMemoryView_Type": "space.w_memoryview", "PyBaseObject_Type": "space.w_object", - 'PyNone_Type': 'space.type(space.w_None)', - 'PyNotImplemented_Type': 'space.type(space.w_NotImplemented)', + '_PyNone_Type': 'space.type(space.w_None)', + '_PyNotImplemented_Type': 'space.type(space.w_NotImplemented)', 'PyCell_Type': 'space.gettypeobject(Cell.typedef)', 'PyModule_Type': 'space.gettypeobject(Module.typedef)', 'PyProperty_Type': 'space.gettypeobject(W_Property.typedef)', @@ -1121,9 +1120,6 @@ # of the cpyext module. The C functions are called with no wrapper, # but must not do anything like calling back PyType_Ready(). We # use them just to get a pointer to the PyTypeObjects defined in C. - get_cobject_type = rffi.llexternal('_%s_get_cobject_type' % prefix, - [], PyTypeObjectPtr, - compilation_info=eci, _nowrapper=True) get_capsule_type = rffi.llexternal('_%s_get_capsule_type' % prefix, [], PyTypeObjectPtr, compilation_info=eci, _nowrapper=True) @@ -1134,7 +1130,6 @@ def init_types(space): from pypy.module.cpyext.typeobject import py_type_ready from pypy.module.sys.interp_encoding import getfilesystemencoding - py_type_ready(space, get_cobject_type()) py_type_ready(space, get_capsule_type()) s = space.text_w(getfilesystemencoding(space)) setdefenc(rffi.str2charp(s, track_allocation=False)) # "leaks" @@ -1495,7 +1490,6 @@ source_dir / "pythonrun.c", source_dir / "sysmodule.c", source_dir / "complexobject.c", - source_dir / "cobject.c", source_dir / "structseq.c", source_dir / "capsule.c", source_dir / "pysignals.c", @@ -1549,7 +1543,6 @@ if sys.platform == 'win32': get_pythonapi_source = ''' - #include RPY_EXTERN HANDLE pypy_get_pythonapi_handle() { MEMORY_BASIC_INFORMATION mi; @@ -1563,6 +1556,9 @@ } ''' separate_module_sources.append(get_pythonapi_source) + kwds['post_include_bits'] = ['#include ', + 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', + ] eci = ExternalCompilationInfo( include_dirs=include_dirs, @@ -1673,7 +1669,11 @@ try: ll_libname = rffi.str2charp(path) try: - dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) + if WIN32: + # Allow other DLLs in the same directory with "path" + dll = rdynload.dlopenex(ll_libname) + else: + dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject +from pypy.module.cpyext.frameobject import PyFrameObject from pypy.module.__builtin__ import compiling PyCompilerFlags = cpython_struct( @@ -58,6 +59,11 @@ return None return caller.get_w_globals() # borrowed ref + at cpython_api([], PyFrameObject, error=CANNOT_FAIL, result_borrowed=True) +def PyEval_GetFrame(space): + caller = space.getexecutioncontext().gettopframe_nohidden() + return caller # borrowed ref, may be null + @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): """This is a simplified interface to PyEval_EvalCodeEx(), with just diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -123,7 +123,6 @@ #include "memoryobject.h" #include "eval.h" #include "pymem.h" -#include "pycobject.h" #include "pycapsule.h" #include "bytesobject.h" #include "sliceobject.h" diff --git a/pypy/module/cpyext/include/abstract.h b/pypy/module/cpyext/include/abstract.h --- a/pypy/module/cpyext/include/abstract.h +++ b/pypy/module/cpyext/include/abstract.h @@ -4,6 +4,15 @@ extern "C" { #endif + PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, char *key); + + /* + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ + + /* new buffer API */ #define PyObject_CheckBuffer(obj) \ @@ -27,6 +36,27 @@ /* Releases a Py_buffer obtained from getbuffer ParseTuple's s*. */ +/* Mapping protocol:*/ + + /* implemented as a macro: + + int PyMapping_DelItemString(PyObject *o, char *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItemString(O,K) PyObject_DelItemString((O),(K)) + + /* implemented as a macro: + + int PyMapping_DelItem(PyObject *o, PyObject *key); + + Remove the mapping for object, key, from the object *o. + Returns -1 on failure. This is equivalent to + the Python statement: del o[key]. + */ +#define PyMapping_DelItem(O,K) PyObject_DelItem((O),(K)) #ifdef __cplusplus } diff --git a/pypy/module/cpyext/include/boolobject.h b/pypy/module/cpyext/include/boolobject.h --- a/pypy/module/cpyext/include/boolobject.h +++ b/pypy/module/cpyext/include/boolobject.h @@ -1,5 +1,4 @@ - -/* Bool object interface */ +/* Boolean object interface */ #ifndef Py_BOOLOBJECT_H #define Py_BOOLOBJECT_H @@ -7,15 +6,19 @@ extern "C" { #endif -#define Py_False ((PyObject *) &_Py_ZeroStruct) +#define PyBool_Check(x) (Py_TYPE(x) == &PyBool_Type) + +/* Py_False and Py_True are the only two bools in existence. +Don't forget to apply Py_INCREF() when returning either!!! */ + +/* Use these macros */ +#define Py_False ((PyObject *) &_Py_FalseStruct) #define Py_True ((PyObject *) &_Py_TrueStruct) /* Macros for returning Py_True or Py_False, respectively */ #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False -#define PyBool_Check(op) ((op)->ob_type == &PyBool_Type) - #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/pycobject.h b/pypy/module/cpyext/include/pycobject.h deleted file mode 100644 --- a/pypy/module/cpyext/include/pycobject.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef Py_COBJECT_H -#define Py_COBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -PyAPI_DATA(PyTypeObject) PyCObject_Type; - -#define PyCObject_Check(op) ((op)->ob_type == &PyCObject_Type) - -/* Create a PyCObject from a pointer to a C object and an optional - destructor function. If the second argument is non-null, then it - will be called with the first argument if and when the PyCObject is - destroyed. - -*/ -PyAPI_FUNC(PyObject *) PyCObject_FromVoidPtr( - void *cobj, void (*destruct)(void*)); - - -/* Create a PyCObject from a pointer to a C object, a description object, - and an optional destructor function. If the third argument is non-null, - then it will be called with the first and second arguments if and when - the PyCObject is destroyed. -*/ -PyAPI_FUNC(PyObject *) PyCObject_FromVoidPtrAndDesc( - void *cobj, void *desc, void (*destruct)(void*,void*)); - -/* Retrieve a pointer to a C object from a PyCObject. */ -PyAPI_FUNC(void *) PyCObject_AsVoidPtr(PyObject *); - -/* Retrieve a pointer to a description object from a PyCObject. */ -PyAPI_FUNC(void *) PyCObject_GetDesc(PyObject *); - -/* Import a pointer to a C object from a module using a PyCObject. */ -PyAPI_FUNC(void *) PyCObject_Import(const char *module_name, const char *cobject_name); - -/* Modify a C object. Fails (==0) if object has a destructor. */ -PyAPI_FUNC(int) PyCObject_SetVoidPtr(PyObject *self, void *cobj); - - -#if (PY_VERSION_HEX >= 0x02060000 || defined(Py_BUILD_CORE)) -typedef struct { - PyObject_HEAD - void *cobject; - void *desc; - void (*destructor)(void *); -} PyCObject; -#endif - -PyAPI_FUNC(PyTypeObject *) _Py_get_cobject_type(void); - -#ifdef __cplusplus -} -#endif -#endif /* !Py_COBJECT_H */ diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -41,19 +41,6 @@ w_mode = space.newtext(rffi.charp2str(mode)) return space.call_method(space.builtin, 'open', w_filename, w_mode) - at cpython_api([FILEP, CONST_STRING, CONST_STRING, rffi.VOIDP], PyObject) -def PyFile_FromFile(space, fp, name, mode, close): - """Create a new PyFileObject from the already-open standard C file - pointer, fp. The function close will be called when the file should be - closed. Return NULL on failure.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.INT_real], lltype.Void) -def PyFile_SetBufSize(space, w_file, n): - """Available on systems with setvbuf() only. This should only be called - immediately after file object creation.""" - raise NotImplementedError - @cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) def PyFile_WriteString(space, s, w_p): """Write string s to file object p. Return 0 on success or -1 on @@ -75,9 +62,3 @@ w_str = space.repr(w_obj) space.call_method(w_p, "write", w_str) return 0 - - at cpython_api([PyObject], PyObject) -def PyFile_Name(space, w_p): - """Return the name of the file specified by p as a string object.""" - w_name = space.getattr(w_p, space.newtext("name")) - return w_name # borrowed ref, should be a W_StringObject from the file diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c --- a/pypy/module/cpyext/src/abstract.c +++ b/pypy/module/cpyext/src/abstract.c @@ -23,6 +23,23 @@ /* Operations on any object */ int +PyObject_DelItemString(PyObject *o, char *key) +{ + PyObject *okey; + int ret; + + if (o == NULL || key == NULL) { + null_error(); + return -1; + } + okey = PyUnicode_FromString(key); + if (okey == NULL) + return -1; + ret = PyObject_DelItem(o, okey); + Py_DECREF(okey); + return ret; +} +int PyObject_CheckReadBuffer(PyObject *obj) { PyBufferProcs *pb = obj->ob_type->tp_as_buffer; @@ -96,6 +113,20 @@ return 0; } +/* Buffer C-API for Python 3.0 */ + +int +PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) +{ + if (!PyObject_CheckBuffer(obj)) { + PyErr_Format(PyExc_TypeError, + "'%100s' does not have the buffer interface", + Py_TYPE(obj)->tp_name); + return -1; + } + return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); +} + void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) { @@ -111,6 +142,7 @@ return (void*)pointer; } + void _Py_add_one_to_index_F(int nd, Py_ssize_t *index, const Py_ssize_t *shape) { @@ -253,19 +285,6 @@ -/* Buffer C-API for Python 3.0 */ - -int -PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) -{ - if (!PyObject_CheckBuffer(obj)) { - PyErr_Format(PyExc_TypeError, - "'%100s' does not have the buffer interface", - Py_TYPE(obj)->tp_name); - return -1; - } - return (*(obj->ob_type->tp_as_buffer->bf_getbuffer))(obj, view, flags); -} void PyBuffer_Release(Py_buffer *view) @@ -427,6 +446,7 @@ return retval; } + static PyObject * objargs_mktuple(va_list va) { diff --git a/pypy/module/cpyext/src/cobject.c b/pypy/module/cpyext/src/cobject.c deleted file mode 100644 --- a/pypy/module/cpyext/src/cobject.c +++ /dev/null @@ -1,162 +0,0 @@ - -/* Wrap void* pointers to be passed between C modules */ - -#include "Python.h" - - -/* Declarations for objects of type PyCObject */ - -typedef void (*destructor1)(void *); -typedef void (*destructor2)(void *, void*); - -PyObject * -PyCObject_FromVoidPtr(void *cobj, void (*destr)(void *)) -{ - PyCObject *self; - - self = PyObject_NEW(PyCObject, &PyCObject_Type); - if (self == NULL) - return NULL; - self->cobject=cobj; - self->destructor=destr; - self->desc=NULL; - - return (PyObject *)self; -} - -PyObject * -PyCObject_FromVoidPtrAndDesc(void *cobj, void *desc, - void (*destr)(void *, void *)) -{ - PyCObject *self; - - if (!desc) { - PyErr_SetString(PyExc_TypeError, - "PyCObject_FromVoidPtrAndDesc called with null" - " description"); - return NULL; - } - self = PyObject_NEW(PyCObject, &PyCObject_Type); - if (self == NULL) - return NULL; - self->cobject = cobj; - self->destructor = (destructor1)destr; - self->desc = desc; - - return (PyObject *)self; -} - -void * -PyCObject_AsVoidPtr(PyObject *self) -{ - if (self) { - if (PyCapsule_CheckExact(self)) { - const char *name = PyCapsule_GetName(self); - return (void *)PyCapsule_GetPointer(self, name); - } - if (self->ob_type == &PyCObject_Type) - return ((PyCObject *)self)->cobject; - PyErr_SetString(PyExc_TypeError, - "PyCObject_AsVoidPtr with non-C-object"); - } - if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, - "PyCObject_AsVoidPtr called with null pointer"); - return NULL; -} - -void * -PyCObject_GetDesc(PyObject *self) -{ - if (self) { - if (self->ob_type == &PyCObject_Type) - return ((PyCObject *)self)->desc; - PyErr_SetString(PyExc_TypeError, - "PyCObject_GetDesc with non-C-object"); - } - if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, - "PyCObject_GetDesc called with null pointer"); - return NULL; -} - -void * -PyCObject_Import(const char *module_name, const char *name) -{ - PyObject *m, *c; - void *r = NULL; - - if ((m = PyImport_ImportModule(module_name))) { - if ((c = PyObject_GetAttrString(m,name))) { - r = PyCObject_AsVoidPtr(c); - Py_DECREF(c); - } - Py_DECREF(m); - } - return r; -} - -int -PyCObject_SetVoidPtr(PyObject *self, void *cobj) -{ - PyCObject* cself = (PyCObject*)self; - if (cself == NULL || !PyCObject_Check(cself) || - cself->destructor != NULL) { - PyErr_SetString(PyExc_TypeError, - "Invalid call to PyCObject_SetVoidPtr"); - return 0; - } - cself->cobject = cobj; - return 1; -} - -static void -PyCObject_dealloc(PyCObject *self) -{ - if (self->destructor) { - if(self->desc) - ((destructor2)(self->destructor))(self->cobject, self->desc); - else - (self->destructor)(self->cobject); - } - PyObject_DEL(self); -} - - -PyDoc_STRVAR(PyCObject_Type__doc__, -"C objects to be exported from one extension module to another\n\ -\n\ -C objects are used for communication between extension modules. They\n\ -provide a way for an extension module to export a C interface to other\n\ -extension modules, so that extension modules can use the Python import\n\ -mechanism to link to one another."); - -PyTypeObject PyCObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "PyCObject", /*tp_name*/ - sizeof(PyCObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)PyCObject_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - 0, /*tp_flags*/ - PyCObject_Type__doc__ /*tp_doc*/ -}; - -PyTypeObject *_Py_get_cobject_type(void) -{ - return &PyCObject_Type; -} diff --git a/pypy/module/cpyext/stubs-find-implemented.py b/pypy/module/cpyext/stubs-find-implemented.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/stubs-find-implemented.py @@ -0,0 +1,21 @@ +import re +import os + + +for line in open('stubs.py'): + if not line.strip(): + continue + if line.startswith(' '): + continue + if line.startswith('#'): + continue + if line.startswith('@cpython_api'): + continue + if line.endswith(' = rffi.VOIDP\n'): + continue + + #print line.rstrip() + m = re.match(r"def ([\w\d_]+)[(]", line) + assert m, line + funcname = m.group(1) + os.system('grep -w %s [a-r]*.py s[a-s]*.py str*.py stubsa*.py sy*.py [t-z]*.py' % funcname) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1,29 +1,10 @@ -from pypy.module.cpyext.api import ( - cpython_api, PyObject, PyObjectP, CANNOT_FAIL - ) -from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex -from rpython.rtyper.lltypesystem import rffi, lltype +#----this file is not imported, only here for reference---- -CWCHARPP = lltype.Ptr(lltype.Array(rffi.CWCHARP, hints={'nolength': True})) - -# we don't really care -PyTypeObjectPtr = rffi.VOIDP -Py_ssize_t = rffi.SSIZE_T -PyModuleDef = rffi.VOIDP -PyMethodDef = rffi.VOIDP -PyGetSetDef = rffi.VOIDP -PyMemberDef = rffi.VOIDP -va_list = rffi.VOIDP -wrapperbase = rffi.VOIDP -FILE = rffi.VOIDP -PyFrameObject = rffi.VOIDP -_inittab = rffi.VOIDP -PyThreadState = rffi.VOIDP -PyInterpreterState = rffi.VOIDP -Py_UNICODE = lltype.UniChar -PyCompilerFlags = rffi.VOIDP -struct_node = rffi.VOIDP -Py_tracefunc = rffi.VOIDP +#from pypy.module.cpyext.api import ( +# cpython_api, PyObject, PyObjectP, CANNOT_FAIL +# ) +#from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex +#from rpython.rtyper.lltypesystem import rffi, lltype @cpython_api([rffi.CCHARP], Py_ssize_t, error=-1) @@ -228,39 +209,6 @@ this method returns zero and sets errno to EDOM.""" raise NotImplementedError - at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP) -def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): - """Convert a double val to a string using supplied - format_code, precision, and flags. - - format_code must be one of 'e', 'E', 'f', 'F', - 'g', 'G' or 'r'. For 'r', the supplied precision - must be 0 and is ignored. The 'r' format code specifies the - standard repr() format. - - flags can be zero or more of the values Py_DTSF_SIGN, - Py_DTSF_ADD_DOT_0, or Py_DTSF_ALT, or-ed together: - - Py_DTSF_SIGN means to always precede the returned string with a sign - character, even if val is non-negative. - - Py_DTSF_ADD_DOT_0 means to ensure that the returned string will not look - like an integer. - - Py_DTSF_ALT means to apply "alternate" formatting rules. See the - documentation for the PyOS_snprintf() '#' specifier for - details. - - If ptype is non-NULL, then the value it points to will be set to one of - Py_DTST_FINITE, Py_DTST_INFINITE, or Py_DTST_NAN, signifying that - val is a finite number, an infinite number, or not a number, respectively. - - The return value is a pointer to buffer with the converted string or - NULL if the conversion failed. The caller is responsible for freeing the - returned string by calling PyMem_Free(). - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP, rffi.CCHARP], rffi.CCHARP) def PyOS_stricmp(space, s1, s2): """Case insensitive comparison of strings. The function works almost @@ -275,24 +223,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_Check(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType or a subtype of - PyDateTime_TZInfoType. ob must not be NULL. - """ - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_CheckExact(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType. ob must not be - NULL. - """ - raise NotImplementedError - - at cpython_api([PyTypeObjectPtr, PyGetSetDef], PyObject) -def PyDescr_NewGetSet(space, type, getset): - raise NotImplementedError - @cpython_api([PyTypeObjectPtr, PyMemberDef], PyObject) def PyDescr_NewMember(space, type, meth): raise NotImplementedError @@ -483,31 +413,6 @@ 0 on success, -1 on failure.""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=-1) -def Py_ReprEnter(space, object): - """Called at the beginning of the tp_repr implementation to - detect cycles. - - If the object has already been processed, the function returns a - positive integer. In that case the tp_repr implementation - should return a string object indicating a cycle. As examples, - dict objects return {...} and list objects - return [...]. - - The function will return a negative integer if the recursion limit - is reached. In that case the tp_repr implementation should - typically return NULL. - - Otherwise, the function returns zero and the tp_repr - implementation can continue normally.""" - raise NotImplementedError - - at cpython_api([PyObject], lltype.Void) -def Py_ReprLeave(space, object): - """Ends a Py_ReprEnter(). Must be called once for each - invocation of Py_ReprEnter() that returns zero.""" - raise NotImplementedError - @cpython_api([rffi.INT_real, rffi.CCHARP, rffi.CCHARP, rffi.INT_real, rffi.CCHARP, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFd(space, fd, name, mode, buffering, encoding, errors, newline, closefd): """Create a Python file object from the file descriptor of an already @@ -1295,39 +1200,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.VOIDP) -def PyModule_GetState(space, module): - """Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or NULL. See - PyModuleDef.m_size.""" - raise NotImplementedError - - at cpython_api([PyObject], PyModuleDef) -def PyModule_GetDef(space, module): - """Return a pointer to the PyModuleDef struct from which the module was - created, or NULL if the module wasn't created with - PyModule_Create().""" - raise NotImplementedError - - - at cpython_api([PyModuleDef], PyObject) -def PyModule_Create(space, module): - """Create a new module object, given the definition in module. This behaves - like PyModule_Create2() with module_api_version set to - PYTHON_API_VERSION.""" - raise NotImplementedError - - - at cpython_api([PyModuleDef, rffi.INT_real], PyObject) -def PyModule_Create2(space, module, module_api_version): - """Create a new module object, given the definition in module, assuming the - API version module_api_version. If that version does not match the version - of the running interpreter, a RuntimeWarning is emitted. - - Most uses of this function should be using PyModule_Create() - instead; only use this if you are sure you need it.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.INT_real], PyObject) def PyNumber_ToBase(space, n, base): """Returns the integer n converted to base base as a string. The base @@ -1337,23 +1209,6 @@ PyNumber_Index() first.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyObject_Bytes(space, o): - """ - Compute a bytes representation of object o. NULL is returned on - failure and a bytes object on success. This is equivalent to the Python - expression bytes(o), when o is not an integer. Unlike bytes(o), - a TypeError is raised when o is an integer instead of a zero-initialized - bytes object.""" - raise NotImplementedError - - at cpython_api([], PyFrameObject) -def PyEval_GetFrame(space): - """Return the current thread state's frame, which is NULL if no frame is - currently executing.""" - raise NotImplementedError - borrow_from() - @cpython_api([PyFrameObject], rffi.INT_real, error=-1) def PyFrame_GetLineNumber(space, frame): """Return the line number that frame is currently executing.""" diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -510,3 +510,15 @@ assert run_async(list1()) == ([], [0, 1, 2]) assert run_async(list2()) == ([], [0, 1, 2]) """ + + def test_getframe(self): + import sys + module = self.import_extension('foo', [ + ("getframe1", "METH_NOARGS", + """ + PyFrameObject *x = PyEval_GetFrame(); + Py_INCREF(x); + return (PyObject *)x; + """),], prologue="#include \n") + res = module.getframe1() + assert res is sys._getframe(0) diff --git a/pypy/module/cpyext/test/test_pycobject.py b/pypy/module/cpyext/test/test_pycobject.py deleted file mode 100644 --- a/pypy/module/cpyext/test/test_pycobject.py +++ /dev/null @@ -1,30 +0,0 @@ -import py -from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase - -class AppTestStringObject(AppTestCpythonExtensionBase): - def test_pycobject_import(self): - module = self.import_extension('foo', [ - ("set_ptr", "METH_O", - """ - PyObject *pointer, *module; - void *ptr = PyLong_AsVoidPtr(args); - if (PyErr_Occurred()) return NULL; - pointer = PyCObject_FromVoidPtr(ptr, NULL); - if (PyErr_Occurred()) return NULL; - module = PyImport_ImportModule("foo"); - PyModule_AddObject(module, "_ptr", pointer); - Py_DECREF(module); - if (PyErr_Occurred()) return NULL; - Py_RETURN_NONE; - """), - ("get_ptr", "METH_NOARGS", - """ - void *ptr = PyCObject_Import("foo", "_ptr"); - if (PyErr_Occurred()) return NULL; - return PyLong_FromVoidPtr(ptr); - """)]) - module.set_ptr(1234) - assert "PyCObject object" in str(module._ptr) - import gc; gc.collect() - assert module.get_ptr() == 1234 - del module._ptr diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -231,7 +231,9 @@ if os.name == 'nt': _source_code = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include #include diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -151,7 +151,7 @@ if getattr(func, '_elidable_function_', False): raise TypeError("it does not make sense for %s to be both elidable and unroll_safe" % func) if not getattr(func, '_jit_look_inside_', True): - raise TypeError("it does not make sense for %s to be both elidable and dont_look_inside" % func) + raise TypeError("it does not make sense for %s to be both unroll_safe and dont_look_inside" % func) func._jit_unroll_safe_ = True return func diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -233,6 +233,15 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenex(name): + res = rwin32.LoadLibraryExA(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -835,7 +835,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = rwin32.get_osfhandle(fileno) + fh = rffi.cast(HANDLE, rwin32.get_osfhandle(fileno)) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,7 +137,10 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', + 'RPY_EXTERN void* enter_suppress_iph();', + 'RPY_EXTERN void exit_suppress_iph(void* handle);', + ] else: separate_module_sources = [] post_include_bits = [] @@ -235,7 +238,8 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', + 'direct.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): @@ -730,16 +734,21 @@ length = rwin32.MAX_PATH + 1 traits = _preferred_traits(path) win32traits = make_win32_traits(traits) - with traits.scoped_alloc_buffer(length) as buf: - res = win32traits.GetFullPathName( - traits.as_str0(path), rffi.cast(rwin32.DWORD, length), - buf.raw, lltype.nullptr(win32traits.LPSTRP.TO)) - if res == 0: - raise rwin32.lastSavedWindowsError("_getfullpathname failed") - result = buf.str(intmask(res)) - assert result is not None - result = rstring.assert_str0(result) - return result + while True: # should run the loop body maximum twice + with traits.scoped_alloc_buffer(length) as buf: + res = win32traits.GetFullPathName( + traits.as_str0(path), rffi.cast(rwin32.DWORD, length), + buf.raw, lltype.nullptr(win32traits.LPSTRP.TO)) + res = intmask(res) + if res == 0: + raise rwin32.lastSavedWindowsError("_getfullpathname failed") + if res >= length: + length = res + 1 + continue + result = buf.str(res) + assert result is not None + result = rstring.assert_str0(result) + return result c_getcwd = external(UNDERSCORE_ON_WIN32 + 'getcwd', [rffi.CCHARP, rffi.SIZE_T], rffi.CCHARP, diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1777,8 +1777,6 @@ "truncated input", s, pos, size) result.append(res) - if pos > size - unicode_bytes: - break continue t = r_uint(0) h = 0 diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -20,7 +20,7 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h', 'stdio.h', 'stdlib.h'], + includes = ['windows.h', 'stdio.h', 'stdlib.h', 'io.h'], libraries = ['kernel32'], ) else: @@ -113,6 +113,7 @@ MB_ERR_INVALID_CHARS ERROR_NO_UNICODE_TRANSLATION WC_NO_BEST_FIT_CHARS STD_INPUT_HANDLE STD_OUTPUT_HANDLE STD_ERROR_HANDLE HANDLE_FLAG_INHERIT FILE_TYPE_CHAR + LOAD_WITH_ALTERED_SEARCH_PATH """ from rpython.translator.platform import host_factory static_platform = host_factory() @@ -195,12 +196,28 @@ GetModuleHandle = winexternal('GetModuleHandleA', [rffi.CCHARP], HMODULE) LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) + def wrap_loadlibraryex(func): + def loadlibrary(name, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + # Requires a full path name with '/' -> '\\' + return func(name, NULL_HANDLE, flags) + return loadlibrary + + _LoadLibraryExA = winexternal('LoadLibraryExA', + [rffi.CCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExA = wrap_loadlibraryex(_LoadLibraryExA) + LoadLibraryW = winexternal('LoadLibraryW', [rffi.CWCHARP], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + _LoadLibraryExW = winexternal('LoadLibraryExW', + [rffi.CWCHARP, HANDLE, DWORD], HMODULE, + save_err=rffi.RFFI_SAVE_LASTERROR) + LoadLibraryExW = wrap_loadlibraryex(_LoadLibraryExW) GetProcAddress = winexternal('GetProcAddress', [HMODULE, rffi.CCHARP], rffi.VOIDP) FreeLibrary = winexternal('FreeLibrary', [HMODULE], BOOL, releasegil=False) - LocalFree = winexternal('LocalFree', [HLOCAL], DWORD) + LocalFree = winexternal('LocalFree', [HLOCAL], HLOCAL) CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL, releasegil=False, save_err=rffi.RFFI_SAVE_LASTERROR) CloseHandle_no_err = winexternal('CloseHandle', [HANDLE], BOOL, @@ -215,12 +232,12 @@ [DWORD, rffi.VOIDP, DWORD, DWORD, rffi.CWCHARP, DWORD, rffi.VOIDP], DWORD) - _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) + _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], rffi.INTP) def get_osfhandle(fd): from rpython.rlib.rposix import FdValidator with FdValidator(fd): - handle = _get_osfhandle(fd) + handle = rffi.cast(HANDLE, _get_osfhandle(fd)) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle diff --git a/rpython/rlib/test/loadtest/loadtest0.dll b/rpython/rlib/test/loadtest/loadtest0.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9bdcc33a1902f8e989d349c49c2cc08e633aa32b GIT binary patch [cut] diff --git a/rpython/rlib/test/loadtest/loadtest1.dll b/rpython/rlib/test/loadtest/loadtest1.dll new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cb83854875c876717371bdf90488eed9c6571f03 GIT binary patch [cut] diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -83,6 +83,14 @@ # the most intriguing failure of ntpath.py should not repeat, here: assert not data.endswith(stuff) + @win_only + def test__getfullpathname_long(self): + stuff = "C:" + "\\abcd" * 100 + py.test.raises(WindowsError, rposix.getfullpathname, stuff) + ustuff = u"C:" + u"\\abcd" * 100 + res = rposix.getfullpathname(ustuff) + assert res == ustuff + def test_getcwd(self): assert rposix.getcwd() == os.getcwd() diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -6,6 +6,44 @@ from rpython.rlib import rwin32 from rpython.tool.udir import udir +loadtest_dir = os.path.dirname(__file__) + '/loadtest' +test1 = os.path.abspath(loadtest_dir + '/loadtest1.dll') +test0 = os.path.abspath(loadtest_dir + '/loadtest0.dll') + +if not os.path.exists(test1) or not os.path.exists(test0): + # This is how the files, which are checked into the repo, were created + from rpython.translator.tool.cbuild import ExternalCompilationInfo + from rpython.translator.platform import platform + from rpython.translator import cdir + if not os.path.exists(loadtest_dir): + os.mkdir(loadtest_dir) + c_file = udir.ensure("test_rwin32", dir=1).join("test0.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + RPY_EXPORTED + int internal_sum(int a, int b) { + return a + b; + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir]) + lib_name = str(platform.compile([c_file], eci, test0[:-4], + standalone=False)) + assert os.path.abspath(lib_name) == os.path.abspath(test0) + + c_file = udir.ensure("test_rwin32", dir=1).join("test1.c") + c_file.write(py.code.Source(''' + #include "src/precommondefs.h" + int internal_sum(int a, int b); + RPY_EXPORTED + int sum(int a, int b) { + return internal_sum(a, b); + } + ''')) + eci = ExternalCompilationInfo(include_dirs=[cdir], + libraries=[loadtest_dir + '/loadtest0']) + lib_name = str(platform.compile([c_file], eci, test1[:-4], + standalone=False, )) + assert os.path.abspath(lib_name) == os.path.abspath(test1) def test_get_osfhandle(): fid = open(str(udir.join('validate_test.txt')), 'w') @@ -28,13 +66,13 @@ "import time;" "time.sleep(10)", ], - ) + ) print proc.pid handle = rwin32.OpenProcess(rwin32.PROCESS_ALL_ACCESS, False, proc.pid) assert rwin32.TerminateProcess(handle, signal.SIGTERM) == 1 rwin32.CloseHandle(handle) assert proc.wait() == signal.SIGTERM - + @py.test.mark.dont_track_allocations('putenv intentionally keeps strings alive') def test_wenviron(): name, value = u'PYPY_TEST_日本', u'foobar日本' @@ -55,3 +93,48 @@ msg = rwin32.FormatErrorW(34) assert type(msg) is unicode assert u'%2' in msg + +def test_loadlibraryA(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibrary(test0) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibrary(test1) + assert not hdll + + assert os.path.exists(test1) + + hdll = rwin32.LoadLibraryExA(test1) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + +def test_loadlibraryW(): + # test0 can be loaded alone, but test1 requires the modified search path + hdll = rwin32.LoadLibraryW(unicode(test0)) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'internal_sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + + hdll = rwin32.LoadLibraryW(unicode(test1)) + assert not hdll + + assert os.path.exists(unicode(test1)) + + hdll = rwin32.LoadLibraryExW(unicode(test1)) + assert hdll + faddr = rwin32.GetProcAddress(hdll, 'sum') + assert faddr + assert rwin32.FreeLibrary(hdll) + +def test_loadlibrary_unicode(): + import shutil + test0u = unicode(udir.join(u'load\u03betest.dll')) + shutil.copyfile(test0, test0u) + hdll = rwin32.LoadLibraryW(test0u) + assert hdll From pypy.commits at gmail.com Fri Oct 12 10:31:47 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 12 Oct 2018 07:31:47 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: fix bad merge from unicode-utf8 Message-ID: <5bc0b053.1c69fb81.72708.5756@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95211:df3f299804fc Date: 2018-10-12 17:31 +0300 http://bitbucket.org/pypy/pypy/changeset/df3f299804fc/ Log: fix bad merge from unicode-utf8 diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -217,6 +217,9 @@ _open_inhcache = rposix.SetNonInheritableCache() +def u2utf8(space, u_str): + return space.newutf8(u_str.encode('utf-8'), len(u_str)) + @unwrap_spec(flags=c_int, mode=c_int, dir_fd=DirFD(rposix.HAVE_OPENAT)) def open(space, w_path, flags, mode=0777, __kwonly__=None, dir_fd=DEFAULT_DIR_FD): From pypy.commits at gmail.com Sun Oct 14 02:32:16 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 13 Oct 2018 23:32:16 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: Test on Posix; avoid using utf-8 and use the filesystem encoding instead Message-ID: <5bc2e2f0.1c69fb81.11a4b.9e9c@mx.google.com> Author: Armin Rigo Branch: cffi_dlopen_unicode Changeset: r95212:8c1e8b2f18b2 Date: 2018-10-14 08:16 +0200 http://bitbucket.org/pypy/pypy/changeset/8c1e8b2f18b2/ Log: Test on Posix; avoid using utf-8 and use the filesystem encoding instead diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -26,16 +26,16 @@ if WIN32 and space.isinstance_w(w_filename, space.w_unicode): fname = space.unicode_w(w_filename) with rffi.scoped_unicode2wcharp(fname) as ll_libname: - fname = fname.encode('utf-8') try: handle = dlopenU(ll_libname, flags) except DLOpenError as e: - raise wrap_dlopenerror(space, e, fname) + w_repr = space.repr(w_filename) + raise wrap_dlopenerror(space, e, space.text_w(w_repr)) else: if space.is_none(w_filename): fname = None elif space.isinstance_w(w_filename, space.w_unicode): - fname = space.unicode_w(w_filename).encode('utf-8') + fname = space.fsencode_w(w_filename) else: fname = space.text_w(w_filename) with rffi.scoped_str2charp(fname) as ll_libname: diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -28,16 +28,16 @@ if WIN32 and space.isinstance_w(w_filename, space.w_unicode): fname = space.unicode_w(w_filename) with rffi.scoped_unicode2wcharp(fname) as ll_libname: - fname = fname.encode('utf-8') try: handle = dlopenU(ll_libname, flags) except DLOpenError as e: - raise wrap_dlopenerror(space, e, fname) + w_repr = space.repr(w_filename) + raise wrap_dlopenerror(space, e, space.text_w(w_repr)) else: if space.is_none(w_filename): fname = None elif space.isinstance_w(w_filename, space.w_unicode): - fname = space.unicode_w(w_filename).encode('utf-8') + fname = space.fsencode_w(w_filename) else: fname = space.text_w(w_filename) with rffi.scoped_str2charp(fname) as ll_libname: diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,5 +1,5 @@ import py -import sys, shutil +import sys, shutil, os from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache @@ -46,8 +46,15 @@ outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) if WIN32: - # non-windows need utf8 locales to deal with unicode filenames - outputfileUname = unicode(udir.join(u'load\u03betest.dll')) + unicode_name = u'load\u03betest.dll' + else: + unicode_name = u'load_caf\xe9' + os.path.splitext(outputfilename)[1] + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None # skip the test + if unicode_name is not None: + outputfileUname = os.path.join(unicode(udir), unicode_name) shutil.copyfile(outputfilename, outputfileUname) cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir @@ -119,12 +126,13 @@ assert type(lib.add42) is _cffi_backend.FFI.CData def test_dlopen_unicode(self): - if getattr(self, 'extmodU', None): - import _cffi_backend - self.fix_path() - from re_python_pysrc import ffi - lib = ffi.dlopen(self.extmodU) - assert lib.add42(-10) == 32 + if not getattr(self, 'extmodU', None): + skip("no unicode file name") + import _cffi_backend + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 def test_dlclose(self): import _cffi_backend From pypy.commits at gmail.com Sun Oct 14 03:31:42 2018 From: pypy.commits at gmail.com (arigo) Date: Sun, 14 Oct 2018 00:31:42 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: Share the code. Fix for Windows, still untested Message-ID: <5bc2f0de.1c69fb81.22161.1295@mx.google.com> Author: Armin Rigo Branch: cffi_dlopen_unicode Changeset: r95213:13735d96ce4f Date: 2018-10-14 09:26 +0200 http://bitbucket.org/pypy/pypy/changeset/13735d96ce4f/ Log: Share the code. Fix for Windows, still untested diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,50 +1,21 @@ -import sys from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.interpreter.error import oefmt -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from pypy.module._cffi_backend.parse_c_type import ( _CFFI_OPCODE_T, GLOBAL_S, CDL_INTCONST_S, STRUCT_UNION_S, FIELD_S, ENUM_S, TYPENAME_S, ll_set_cdl_realize_global_int) from pypy.module._cffi_backend.realize_c_type import getop from pypy.module._cffi_backend.lib_obj import W_LibObject -from pypy.module._cffi_backend import cffi_opcode, cffi1_module - -if sys.platform == 'win32': - from rpython.rlib.rdynload import dlopenU - WIN32 = True -else: - WIN32 = False +from pypy.module._cffi_backend import cffi_opcode, cffi1_module, misc class W_DlOpenLibObject(W_LibObject): def __init__(self, ffi, w_filename, flags): space = ffi.space - if WIN32 and space.isinstance_w(w_filename, space.w_unicode): - fname = space.unicode_w(w_filename) - with rffi.scoped_unicode2wcharp(fname) as ll_libname: - try: - handle = dlopenU(ll_libname, flags) - except DLOpenError as e: - w_repr = space.repr(w_filename) - raise wrap_dlopenerror(space, e, space.text_w(w_repr)) - else: - if space.is_none(w_filename): - fname = None - elif space.isinstance_w(w_filename, space.w_unicode): - fname = space.fsencode_w(w_filename) - else: - fname = space.text_w(w_filename) - with rffi.scoped_str2charp(fname) as ll_libname: - if fname is None: - fname = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, fname) + fname, handle = misc.dlopen_w(space, w_filename, flags) W_LibObject.__init__(self, ffi, fname) self.libhandle = handle self.register_finalizer(space) diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -1,23 +1,16 @@ from __future__ import with_statement -import sys from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType - -if sys.platform == 'win32': - from rpython.rlib.rdynload import dlopenU - WIN32 = True -else: - WIN32 = False +from pypy.module._cffi_backend import misc class W_Library(W_Root): @@ -25,30 +18,7 @@ def __init__(self, space, w_filename, flags): self.space = space - if WIN32 and space.isinstance_w(w_filename, space.w_unicode): - fname = space.unicode_w(w_filename) - with rffi.scoped_unicode2wcharp(fname) as ll_libname: - try: - handle = dlopenU(ll_libname, flags) - except DLOpenError as e: - w_repr = space.repr(w_filename) - raise wrap_dlopenerror(space, e, space.text_w(w_repr)) - else: - if space.is_none(w_filename): - fname = None - elif space.isinstance_w(w_filename, space.w_unicode): - fname = space.fsencode_w(w_filename) - else: - fname = space.text_w(w_filename) - with rffi.scoped_str2charp(fname) as ll_libname: - if fname is None: - fname = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, fname) - self.handle = handle - self.name = fname + self.name, self.handle = misc.dlopen_w(space, w_filename, flags) self.register_finalizer(space) def _finalize_(self): diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,14 +1,23 @@ from __future__ import with_statement +import sys from pypy.interpreter.error import OperationError, oefmt +from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rlib import jit from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.rdynload import dlopen, DLOpenError from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + # ____________________________________________________________ @@ -383,3 +392,30 @@ ptr = rffi.cast(rffi.FLOATP, source) for i in range(len(float_list)): float_list[i] = rffi.cast(lltype.Float, ptr[i]) + +# ____________________________________________________________ + +def dlopen_w(space, w_filename, flags): + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.text_w(space.repr(w_filename)) + unicode_name = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(unicode_name) as ll_libname: + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + elif space.isinstance_w(w_filename, space.w_unicode): + fname = space.fsencode_w(w_filename) + else: + fname = space.text_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + return fname, handle diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -52,7 +52,7 @@ try: unicode_name.encode(sys.getfilesystemencoding()) except UnicodeEncodeError: - unicode_name = None # skip the test + unicode_name = None # skip test_dlopen_unicode if unicode_name is not None: outputfileUname = os.path.join(unicode(udir), unicode_name) shutil.copyfile(outputfilename, outputfileUname) From pypy.commits at gmail.com Sun Oct 14 03:42:39 2018 From: pypy.commits at gmail.com (arigo) Date: Sun, 14 Oct 2018 00:42:39 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: hg merge default Message-ID: <5bc2f36f.1c69fb81.c8d25.a056@mx.google.com> Author: Armin Rigo Branch: cffi_dlopen_unicode Changeset: r95214:b2e4adfaa71e Date: 2018-10-14 09:40 +0200 http://bitbucket.org/pypy/pypy/changeset/b2e4adfaa71e/ Log: hg merge default diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -95,7 +95,9 @@ if os.name == 'nt': do_includes = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include static void _cffi_init(void); diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -92,8 +92,11 @@ if sys.platform == 'win32': dash = '_' + WIN32 = True else: dash = '' + WIN32 = False + def fclose(fp): try: @@ -1537,7 +1540,6 @@ if sys.platform == 'win32': get_pythonapi_source = ''' - #include RPY_EXTERN HANDLE pypy_get_pythonapi_handle() { MEMORY_BASIC_INFORMATION mi; @@ -1551,6 +1553,9 @@ } ''' separate_module_sources.append(get_pythonapi_source) + kwds['post_include_bits'] = ['#include ', + 'RPY_EXTERN HANDLE pypy_get_pythonapi_handle();', + ] eci = ExternalCompilationInfo( include_dirs=include_dirs, @@ -1656,7 +1661,11 @@ try: ll_libname = rffi.str2charp(path) try: - dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) + if WIN32: + # Allow other DLLs in the same directory with "path" + dll = rdynload.dlopenex(ll_libname) + else: + dll = rdynload.dlopen(ll_libname, space.sys.dlopenflags) finally: lltype.free(ll_libname, flavor='raw') except rdynload.DLOpenError as e: diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -9,6 +9,7 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject +from pypy.module.cpyext.frameobject import PyFrameObject from pypy.module.__builtin__ import compiling PyCompilerFlags = cpython_struct( @@ -58,6 +59,11 @@ return None return caller.get_w_globals() # borrowed ref + at cpython_api([], PyFrameObject, error=CANNOT_FAIL, result_borrowed=True) +def PyEval_GetFrame(space): + caller = space.getexecutioncontext().gettopframe_nohidden() + return caller # borrowed ref, may be null + @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): """This is a simplified interface to PyEval_EvalCodeEx(), with just diff --git a/pypy/module/cpyext/stubs-find-implemented.py b/pypy/module/cpyext/stubs-find-implemented.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/stubs-find-implemented.py @@ -0,0 +1,21 @@ +import re +import os + + +for line in open('stubs.py'): + if not line.strip(): + continue + if line.startswith(' '): + continue + if line.startswith('#'): + continue + if line.startswith('@cpython_api'): + continue + if line.endswith(' = rffi.VOIDP\n'): + continue + + #print line.rstrip() + m = re.match(r"def ([\w\d_]+)[(]", line) + assert m, line + funcname = m.group(1) + os.system('grep -w %s [a-r]*.py s[a-s]*.py str*.py stubsa*.py sy*.py [t-z]*.py' % funcname) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -1,28 +1,11 @@ -from pypy.module.cpyext.api import ( - cpython_api, PyObject, PyObjectP, CANNOT_FAIL - ) -from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex -from rpython.rtyper.lltypesystem import rffi, lltype +#----this file is not imported, only here for reference---- -# we don't really care -PyTypeObjectPtr = rffi.VOIDP -Py_ssize_t = rffi.SSIZE_T -PyMethodDef = rffi.VOIDP -PyGetSetDef = rffi.VOIDP -PyMemberDef = rffi.VOIDP -va_list = rffi.VOIDP -wrapperbase = rffi.VOIDP -FILE = rffi.VOIDP -PyFileObject = rffi.VOIDP -PyCodeObject = rffi.VOIDP -PyFrameObject = rffi.VOIDP -_inittab = rffi.VOIDP -PyThreadState = rffi.VOIDP -PyInterpreterState = rffi.VOIDP -Py_UNICODE = lltype.UniChar -PyCompilerFlags = rffi.VOIDP -_node = rffi.VOIDP -Py_tracefunc = rffi.VOIDP +#from pypy.module.cpyext.api import ( +# cpython_api, PyObject, PyObjectP, CANNOT_FAIL +# ) +#from pypy.module.cpyext.complexobject import Py_complex_ptr as Py_complex +#from rpython.rtyper.lltypesystem import rffi, lltype + @cpython_api([rffi.CCHARP], Py_ssize_t, error=CANNOT_FAIL) def PyBuffer_SizeFromFormat(space, format): @@ -254,39 +237,6 @@ instead.""" raise NotImplementedError - at cpython_api([rffi.DOUBLE, lltype.Char, rffi.INT_real, rffi.INT_real, rffi.INTP], rffi.CCHARP) -def PyOS_double_to_string(space, val, format_code, precision, flags, ptype): - """Convert a double val to a string using supplied - format_code, precision, and flags. - - format_code must be one of 'e', 'E', 'f', 'F', - 'g', 'G' or 'r'. For 'r', the supplied precision - must be 0 and is ignored. The 'r' format code specifies the - standard repr() format. - - flags can be zero or more of the values Py_DTSF_SIGN, - Py_DTSF_ADD_DOT_0, or Py_DTSF_ALT, or-ed together: - - Py_DTSF_SIGN means to always precede the returned string with a sign - character, even if val is non-negative. - - Py_DTSF_ADD_DOT_0 means to ensure that the returned string will not look - like an integer. - - Py_DTSF_ALT means to apply "alternate" formatting rules. See the - documentation for the PyOS_snprintf() '#' specifier for - details. - - If ptype is non-NULL, then the value it points to will be set to one of - Py_DTST_FINITE, Py_DTST_INFINITE, or Py_DTST_NAN, signifying that - val is a finite number, an infinite number, or not a number, respectively. - - The return value is a pointer to buffer with the converted string or - NULL if the conversion failed. The caller is responsible for freeing the - returned string by calling PyMem_Free(). - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP], rffi.DOUBLE, error=CANNOT_FAIL) def PyOS_ascii_atof(space, nptr): """Convert a string to a double in a locale-independent way. @@ -310,24 +260,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_Check(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType or a subtype of - PyDateTime_TZInfoType. ob must not be NULL. - """ - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyTZInfo_CheckExact(space, ob): - """Return true if ob is of type PyDateTime_TZInfoType. ob must not be - NULL. - """ - raise NotImplementedError - - at cpython_api([PyTypeObjectPtr, PyGetSetDef], PyObject) -def PyDescr_NewGetSet(space, type, getset): - raise NotImplementedError - @cpython_api([PyTypeObjectPtr, PyMemberDef], PyObject) def PyDescr_NewMember(space, type, meth): raise NotImplementedError @@ -1206,14 +1138,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.ULONGLONG, error=-1) -def PyInt_AsUnsignedLongLongMask(space, io): - """Will first attempt to cast the object to a PyIntObject or - PyLongObject, if it is not already one, and then return its value as - unsigned long long, without checking for overflow. - """ - raise NotImplementedError - @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyInt_ClearFreeList(space): """Clear the integer free list. Return the number of items that could not @@ -1333,13 +1257,6 @@ for PyObject_Str().""" raise NotImplementedError - at cpython_api([], PyFrameObject) -def PyEval_GetFrame(space): - """Return the current thread state's frame, which is NULL if no frame is - currently executing.""" - borrow_from() - raise NotImplementedError - @cpython_api([PyFrameObject], rffi.INT_real, error=CANNOT_FAIL) def PyFrame_GetLineNumber(space, frame): """Return the line number that frame is currently executing.""" diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -420,3 +420,15 @@ except StopIteration: pass assert out == [0, 1, 2, 3, 4] + + def test_getframe(self): + import sys + module = self.import_extension('foo', [ + ("getframe1", "METH_NOARGS", + """ + PyFrameObject *x = PyEval_GetFrame(); + Py_INCREF(x); + return (PyObject *)x; + """),], prologue="#include \n") + res = module.getframe1() + assert res is sys._getframe(0) diff --git a/pypy/module/sys/initpath.py b/pypy/module/sys/initpath.py --- a/pypy/module/sys/initpath.py +++ b/pypy/module/sys/initpath.py @@ -183,7 +183,9 @@ if os.name == 'nt': _source_code = r""" +#ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 +#endif #include #include diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -233,6 +233,15 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenex(name): + res = rwin32.LoadLibraryExA(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlopenU(name, mode=-1): # mode is unused on windows, but a consistant signature res = rwin32.LoadLibraryW(name) diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -835,7 +835,7 @@ # assume -1 and 0 both mean invalid file descriptor # to 'anonymously' map memory. if fileno != -1 and fileno != 0: - fh = rwin32.get_osfhandle(fileno) + fh = rffi.cast(HANDLE, rwin32.get_osfhandle(fileno)) # Win9x appears to need us seeked to zero # SEEK_SET = 0 # libc._lseek(fileno, 0, SEEK_SET) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -137,7 +137,10 @@ RPY_EXTERN void exit_suppress_iph(void* handle) {}; #endif ''',] - post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);'] + post_include_bits=['RPY_EXTERN int _PyVerify_fd(int);', + 'RPY_EXTERN void* enter_suppress_iph();', + 'RPY_EXTERN void exit_suppress_iph(void* handle);', + ] else: separate_module_sources = [] post_include_bits = [] @@ -235,7 +238,8 @@ rthread.tlfield_rpy_errno.setraw(_get_errno()) # ^^^ keep fork() up-to-date too, below if _WIN32: - includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h'] + includes = ['io.h', 'sys/utime.h', 'sys/types.h', 'process.h', 'time.h', + 'direct.h'] libraries = [] else: if sys.platform.startswith(('darwin', 'netbsd', 'openbsd')): diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -20,7 +20,7 @@ if WIN32: eci = ExternalCompilationInfo( - includes = ['windows.h', 'stdio.h', 'stdlib.h'], + includes = ['windows.h', 'stdio.h', 'stdlib.h', 'io.h'], libraries = ['kernel32'], ) else: @@ -197,9 +197,9 @@ LoadLibrary = winexternal('LoadLibraryA', [rffi.CCHARP], HMODULE, save_err=rffi.RFFI_SAVE_LASTERROR) def wrap_loadlibraryex(func): - def loadlibrary(name, handle=None, flags=LOAD_WITH_ALTERED_SEARCH_PATH): + def loadlibrary(name, flags=LOAD_WITH_ALTERED_SEARCH_PATH): # Requires a full path name with '/' -> '\\' - return func(name, handle, flags) + return func(name, NULL_HANDLE, flags) return loadlibrary _LoadLibraryExA = winexternal('LoadLibraryExA', @@ -217,7 +217,7 @@ rffi.VOIDP) FreeLibrary = winexternal('FreeLibrary', [HMODULE], BOOL, releasegil=False) - LocalFree = winexternal('LocalFree', [HLOCAL], DWORD) + LocalFree = winexternal('LocalFree', [HLOCAL], HLOCAL) CloseHandle = winexternal('CloseHandle', [HANDLE], BOOL, releasegil=False, save_err=rffi.RFFI_SAVE_LASTERROR) CloseHandle_no_err = winexternal('CloseHandle', [HANDLE], BOOL, @@ -232,12 +232,12 @@ [DWORD, rffi.VOIDP, DWORD, DWORD, rffi.CWCHARP, DWORD, rffi.VOIDP], DWORD) - _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE) + _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], rffi.INTP) def get_osfhandle(fd): from rpython.rlib.rposix import FdValidator with FdValidator(fd): - handle = _get_osfhandle(fd) + handle = rffi.cast(HANDLE, _get_osfhandle(fd)) if handle == INVALID_HANDLE_VALUE: raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle") return handle diff --git a/testrunner/lib_python_tests.py b/testrunner/lib_python_tests.py --- a/testrunner/lib_python_tests.py +++ b/testrunner/lib_python_tests.py @@ -15,6 +15,7 @@ [sys.executable, "pypy/test_all.py", "--pypy=pypy/goal/pypy-c", "--timeout=3600", + "--duration=10", "--resultlog=cpython.log", "lib-python", ] + sys.argv[1:], cwd=rootdir) From pypy.commits at gmail.com Mon Oct 15 09:31:00 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 15 Oct 2018 06:31:00 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: fix iterator stop condition Message-ID: <5bc49694.1c69fb81.401ea.2ab0@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95215:a8c58ad826e8 Date: 2018-10-15 16:30 +0300 http://bitbucket.org/pypy/pypy/changeset/a8c58ad826e8/ Log: fix iterator stop condition diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -276,6 +276,7 @@ for cp in rutf8.Utf8StringIterator(res_8): if cp > 0xFF: errorhandler("strict", 'latin1', msg, s, startindex, index) + raise RuntimeError('error handler should not have returned') result.append(chr(cp)) if index != newindex: # Should be uncommon index = newindex diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py --- a/rpython/rlib/rutf8.py +++ b/rpython/rlib/rutf8.py @@ -760,6 +760,7 @@ self._pos = pos + 1 return ordch1 if pos + 1 >= len(code): + self._pos = pos + 1 return ordch1 ordch2 = ord(code[pos+1]) From pypy.commits at gmail.com Tue Oct 16 07:31:41 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 16 Oct 2018 04:31:41 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: rework logic of space.newtext() to special case None as CPython does Message-ID: <5bc5cc1d.1c69fb81.f5c34.a7f3@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95216:87307b6c75cd Date: 2018-10-16 14:11 +0300 http://bitbucket.org/pypy/pypy/changeset/87307b6c75cd/ Log: rework logic of space.newtext() to special case None as CPython does diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -187,19 +187,18 @@ def descr_new(space, w_unicodetype, w_object=None, w_encoding=None, w_errors=None): if w_object is None: - w_object = W_UnicodeObject.EMPTY - w_obj = w_object - - encoding, errors, allow_surrogates = _get_encoding_and_errors(space, w_encoding, - w_errors) - if encoding is None and errors is None: - # this is very quick if w_obj is already a w_unicode - w_value = unicode_from_object(space, w_obj) + w_value = W_UnicodeObject.EMPTY else: - if space.isinstance_w(w_obj, space.w_unicode): - raise oefmt(space.w_TypeError, + encoding, errors, allow_surrogates = _get_encoding_and_errors(space, + w_encoding, w_errors) + if encoding is None and errors is None: + # this is very quick if w_object is already a w_unicode + w_value = unicode_from_object(space, w_object) + else: + if space.isinstance_w(w_object, space.w_unicode): + raise oefmt(space.w_TypeError, "decoding str is not supported") - w_value = unicode_from_encoded_object(space, w_obj, + w_value = unicode_from_encoded_object(space, w_object, encoding, errors) if space.is_w(w_unicodetype, space.w_unicode): return w_value From pypy.commits at gmail.com Tue Oct 16 07:31:44 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 16 Oct 2018 04:31:44 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: use unicode length, not utf8 length Message-ID: <5bc5cc20.1c69fb81.11a4b.d98c@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95217:b9f98cb79846 Date: 2018-10-16 14:12 +0300 http://bitbucket.org/pypy/pypy/changeset/b9f98cb79846/ Log: use unicode length, not utf8 length diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -72,7 +72,8 @@ def unicode_attach(space, py_obj, w_obj, w_userdata=None): "Fills a newly allocated PyUnicodeObject with a unicode string" value = space.utf8_w(w_obj) - set_wsize(py_obj, len(value)) + length = space.len_w(w_obj) + set_wsize(py_obj, length) set_wbuffer(py_obj, lltype.nullptr(rffi.CWCHARP.TO)) _readify(space, py_obj, value) From pypy.commits at gmail.com Tue Oct 16 07:31:46 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 16 Oct 2018 04:31:46 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: fix cpyext bytesobject tests for python3 Message-ID: <5bc5cc22.1c69fb81.b69b0.83e0@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95218:0c3157538f8c Date: 2018-10-16 14:23 +0300 http://bitbucket.org/pypy/pypy/changeset/0c3157538f8c/ Log: fix cpyext bytesobject tests for python3 diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -1,15 +1,18 @@ # encoding: utf-8 +import pytest from rpython.rtyper.lltypesystem import rffi, lltype +from pypy.interpreter.error import OperationError from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.bytesobject import ( new_empty_str, PyBytesObject, _PyBytes_Resize, PyBytes_Concat, - PyBytes_ConcatAndDel, - _PyBytes_Eq, - _PyBytes_Join) -from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP + _PyBytes_Eq, PyBytes_ConcatAndDel, _PyBytes_Join) +from pypy.module.cpyext.api import (PyObjectP, PyObject, Py_ssize_tP, + Py_buffer, Py_bufferP, generic_cpy_call) from pypy.module.cpyext.pyobject import decref, from_ref, make_ref from pypy.module.cpyext.buffer import PyObject_AsCharBuffer +from pypy.module.cpyext.unicodeobject import (PyUnicode_AsEncodedObject, + PyUnicode_InternFromString, PyUnicode_Format) class AppTestBytesObject(AppTestCpythonExtensionBase): @@ -264,7 +267,8 @@ PyBytes_Concat(space, ptr, space.newbytes('def')) # should not crash lltype.free(ptr, flavor='raw') - def test_ConcatAndDel(self, space): + def test_ConcatAndDel1(self, space): + # XXX remove this or test_ConcatAndDel2 ref1 = make_ref(space, space.newbytes('abc')) ref2 = make_ref(space, space.newbytes('def')) ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw') @@ -314,12 +318,6 @@ w_obj = space.wrap(u"test") with raises_w(space, TypeError): api.PyBytes_FromObject(w_obj) - PyString_AS_STRING(o); - PyString_AS_STRING(u); - - return o; - """)]) - assert module.test_macro_invocations() == '' def test_hash_and_state(self): module = self.import_extension('foo', [ @@ -473,14 +471,14 @@ py_str.c_ob_sval[1] = 'b' py_str.c_ob_sval[2] = 'c' ar[0] = rffi.cast(PyObject, py_str) - _PyString_Resize(space, ar, 3) + _PyBytes_Resize(space, ar, 3) py_str = rffi.cast(PyBytesObject, ar[0]) assert py_str.c_ob_size == 3 assert py_str.c_ob_sval[1] == 'b' assert py_str.c_ob_sval[3] == '\x00' # the same for growing ar[0] = rffi.cast(PyObject, py_str) - _PyString_Resize(space, ar, 10) + _PyBytes_Resize(space, ar, 10) py_str = rffi.cast(PyBytesObject, ar[0]) assert py_str.c_ob_size == 10 assert py_str.c_ob_sval[1] == 'b' @@ -493,61 +491,58 @@ c_buf = py_str.c_ob_type.c_tp_as_buffer assert c_buf py_obj = rffi.cast(PyObject, py_str) - assert generic_cpy_call(space, c_buf.c_bf_getsegcount, - py_obj, lltype.nullptr(Py_ssize_tP.TO)) == 1 - ref = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') - assert generic_cpy_call(space, c_buf.c_bf_getsegcount, - py_obj, ref) == 1 - assert ref[0] == 10 - lltype.free(ref, flavor='raw') - ref = lltype.malloc(rffi.VOIDPP.TO, 1, flavor='raw') - assert generic_cpy_call(space, c_buf.c_bf_getreadbuffer, - py_obj, 0, ref) == 10 + size = rffi.sizeof(Py_buffer) + ref = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) + ref = rffi.cast(Py_bufferP, ref) + assert generic_cpy_call(space, c_buf.c_bf_getbuffer, + py_obj, ref, rffi.cast(rffi.INT_real, 0)) == 0 lltype.free(ref, flavor='raw') decref(space, py_obj) def test_Concat(self, space): - ref = make_ref(space, space.wrap('abc')) + ref = make_ref(space, space.newbytes('abc')) ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw') ptr[0] = ref prev_refcnt = ref.c_ob_refcnt - PyString_Concat(space, ptr, space.wrap('def')) + PyBytes_Concat(space, ptr, space.newbytes('def')) assert ref.c_ob_refcnt == prev_refcnt - 1 - assert space.str_w(from_ref(space, ptr[0])) == 'abcdef' + assert space.utf8_w(from_ref(space, ptr[0])) == 'abcdef' with pytest.raises(OperationError): - PyString_Concat(space, ptr, space.w_None) + PyBytes_Concat(space, ptr, space.w_None) assert not ptr[0] ptr[0] = lltype.nullptr(PyObject.TO) - PyString_Concat(space, ptr, space.wrap('def')) # should not crash + PyBytes_Concat(space, ptr, space.wrap('def')) # should not crash lltype.free(ptr, flavor='raw') - def test_ConcatAndDel(self, space): - ref1 = make_ref(space, space.wrap('abc')) - ref2 = make_ref(space, space.wrap('def')) + def test_ConcatAndDel2(self, space): + # XXX remove this or test_ConcatAndDel1 + ref1 = make_ref(space, space.newbytes('abc')) + ref2 = make_ref(space, space.newbytes('def')) ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw') ptr[0] = ref1 prev_refcnf = ref2.c_ob_refcnt - PyString_ConcatAndDel(space, ptr, ref2) - assert space.str_w(from_ref(space, ptr[0])) == 'abcdef' + PyBytes_ConcatAndDel(space, ptr, ref2) + assert space.utf8_w(from_ref(space, ptr[0])) == 'abcdef' assert ref2.c_ob_refcnt == prev_refcnf - 1 decref(space, ptr[0]) ptr[0] = lltype.nullptr(PyObject.TO) ref2 = make_ref(space, space.wrap('foo')) prev_refcnf = ref2.c_ob_refcnt - PyString_ConcatAndDel(space, ptr, ref2) # should not crash + PyBytes_ConcatAndDel(space, ptr, ref2) # should not crash assert ref2.c_ob_refcnt == prev_refcnf - 1 lltype.free(ptr, flavor='raw') def test_format(self, space): + # XXX move to test_unicodeobject assert "1 2" == space.unwrap( - PyString_Format(space, space.wrap('%s %d'), space.wrap((1, 2)))) + PyUnicode_Format(space, space.wrap('%s %d'), space.wrap((1, 2)))) def test_asbuffer(self, space): bufp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw') - w_text = space.wrap("text") - ref = make_ref(space, w_text) + w_bytes = space.newbytes("text") + ref = make_ref(space, w_bytes) prev_refcnt = ref.c_ob_refcnt assert PyObject_AsCharBuffer(space, ref, bufp, lenp) == 0 assert ref.c_ob_refcnt == prev_refcnt @@ -558,54 +553,49 @@ decref(space, ref) def test_intern(self, space): + # XXX move to test_unicodeobject buf = rffi.str2charp("test") - w_s1 = PyString_InternFromString(space, buf) - w_s2 = PyString_InternFromString(space, buf) + w_s1 = PyUnicode_InternFromString(space, buf) + w_s2 = PyUnicode_InternFromString(space, buf) rffi.free_charp(buf) assert w_s1 is w_s2 def test_AsEncodedObject(self, space): + # XXX move to test_unicodeobject ptr = space.wrap('abc') errors = rffi.str2charp("strict") - encoding = rffi.str2charp("hex") - res = PyString_AsEncodedObject(space, ptr, encoding, errors) - assert space.unwrap(res) == "616263" + encoding = rffi.str2charp("ascii") + res = PyUnicode_AsEncodedObject(space, ptr, encoding, errors) + assert space.unwrap(res) == "abc" - res = PyString_AsEncodedObject(space, + res = PyUnicode_AsEncodedObject(space, ptr, encoding, lltype.nullptr(rffi.CCHARP.TO)) - assert space.unwrap(res) == "616263" + assert space.unwrap(res) == "abc" rffi.free_charp(encoding) encoding = rffi.str2charp("unknown_encoding") with raises_w(space, LookupError): - PyString_AsEncodedObject(space, ptr, encoding, errors) + PyUnicode_AsEncodedObject(space, ptr, encoding, errors) rffi.free_charp(encoding) rffi.free_charp(errors) NULL = lltype.nullptr(rffi.CCHARP.TO) - res = PyString_AsEncodedObject(space, ptr, NULL, NULL) + res = PyUnicode_AsEncodedObject(space, ptr, NULL, NULL) assert space.unwrap(res) == "abc" with raises_w(space, TypeError): - PyString_AsEncodedObject(space, space.wrap(2), NULL, NULL) - - def test_AsDecodedObject(self, space): - w_str = space.wrap('caf\xe9') - encoding = rffi.str2charp("latin-1") - w_res = PyString_AsDecodedObject(space, w_str, encoding, None) - rffi.free_charp(encoding) - assert w_res._utf8 == u"caf\xe9".encode('utf8') + PyUnicode_AsEncodedObject(space, space.wrap(2), NULL, NULL) def test_eq(self, space): - assert 1 == _PyString_Eq( + assert 1 == _PyBytes_Eq( space, space.wrap("hello"), space.wrap("hello")) - assert 0 == _PyString_Eq( + assert 0 == _PyBytes_Eq( space, space.wrap("hello"), space.wrap("world")) def test_join(self, space): w_sep = space.wrap('') w_seq = space.wrap(['a', 'b']) - w_joined = _PyString_Join(space, w_sep, w_seq) + w_joined = _PyBytes_Join(space, w_sep, w_seq) assert space.unwrap(w_joined) == 'ab' From pypy.commits at gmail.com Tue Oct 16 07:31:48 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 16 Oct 2018 04:31:48 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: PyFile_Name was removed from cpython3 Message-ID: <5bc5cc24.1c69fb81.49587.9bd8@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95219:db6acee88a0e Date: 2018-10-16 14:31 +0300 http://bitbucket.org/pypy/pypy/changeset/db6acee88a0e/ Log: PyFile_Name was removed from cpython3 diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -48,13 +48,6 @@ space.call_method(w_file, "close") - def test_file_name(self, space, api): - name = str(udir / "_test_file") - with rffi.scoped_str2charp(name) as filename: - with rffi.scoped_str2charp("wb") as mode: - w_file = api.PyFile_FromString(filename, mode) - assert space.text_w(api.PyFile_Name(w_file)) == name - @pytest.mark.xfail def test_file_setbufsize(self, space, api): api.PyFile_SetBufSize() From pypy.commits at gmail.com Tue Oct 16 13:19:02 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 16 Oct 2018 10:19:02 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: fix to loop infinitely like CPython Message-ID: <5bc61d86.1c69fb81.a520f.70c7@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95220:874dfeb1a872 Date: 2018-10-16 20:18 +0300 http://bitbucket.org/pypy/pypy/changeset/874dfeb1a872/ Log: fix to loop infinitely like CPython diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1548,8 +1548,6 @@ "truncated input", s, pos, size) result.append(res) - if pos > size - unicode_bytes: - break continue t = r_uint(0) h = 0 From pypy.commits at gmail.com Wed Oct 17 02:51:33 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 16 Oct 2018 23:51:33 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: don't encode utf8 in tests Message-ID: <5bc6dbf5.1c69fb81.8c79f.5fb1@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95221:55b18ad54e86 Date: 2018-10-17 09:14 +0300 http://bitbucket.org/pypy/pypy/changeset/55b18ad54e86/ Log: don't encode utf8 in tests diff --git a/pypy/module/_codecs/test/test_locale.py b/pypy/module/_codecs/test/test_locale.py --- a/pypy/module/_codecs/test/test_locale.py +++ b/pypy/module/_codecs/test/test_locale.py @@ -40,8 +40,7 @@ locale_encoder = unicode_encode_locale_surrogateescape utf8_encoder = self.getencoder('utf-8') for val in u'foo', u' 日本', u'\U0001320C': - assert (locale_encoder(val).encode('utf8') == - utf8_encoder(val, 'strict', None)) + assert locale_encoder(val).decode('utf8') == val def test_encode_locale_errorhandler(self): self.setlocale("en_US.UTF-8") @@ -51,7 +50,7 @@ for val in u'foo\udc80bar', u'\udcff\U0001320C': expected = utf8_encoder(val.encode('utf8'), 'surrogateescape', encode_error_handler) - assert locale_encoder(val).encode('utf8') == expected + assert locale_encoder(val) == expected def test_decode_locale(self): self.setlocale("en_US.UTF-8") From pypy.commits at gmail.com Fri Oct 19 12:50:06 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 19 Oct 2018 09:50:06 -0700 (PDT) Subject: [pypy-commit] pypy gc-disable: WIP: start to modify rgc.collect_step to return the old and new GC state Message-ID: <5bca0b3e.1c69fb81.6f3f4.d110@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95222:c91fc7194da4 Date: 2018-10-19 17:48 +0200 http://bitbucket.org/pypy/pypy/changeset/c91fc7194da4/ Log: WIP: start to modify rgc.collect_step to return the old and new GC state diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -19,12 +19,34 @@ def collect_step(): """ - If the GC is incremental, run a single gc-collect-step. Return True when - the major collection is completed. - If the GC is not incremental, do a full collection and return True. + If the GC is incremental, run a single gc-collect-step. + + Return an integer which encodes the starting and ending GC state. Use + rgc.{old_state,new_state,is_done} to decode it. + + If the GC is not incremental, do a full collection and return a value on + which rgc.is_done() return True. """ gc.collect() - return True + return _encode_states(1, 0) + +def _encode_states(oldstate, newstate): + return oldstate << 16 | newstate + +def old_state(val): + return (val & 0xFFFF0000) >> 16 + +def new_state(val): + return val & 0xFFFF + +def is_done(val): + # a collection is considered done when it ends up in the starting state + # (which is usually represented as 0). This logic works for incminimark, + # which is currently the only gc actually used and for which collect_step + # is implemented. In case we add more GC in the future, we might want to + # delegate this logic to the GC itself, but for now it is MUCH simpler to + # just write it in plain RPython. + return old_state(val) != 0 and new_state(val) == 0 def set_max_heap_size(nbytes): """Limit the heap size to n bytes. @@ -174,7 +196,7 @@ def compute_result_annotation(self): from rpython.annotator import model as annmodel - return annmodel.s_Bool + return annmodel.SomeInteger() def specialize_call(self, hop): hop.exception_cannot_occur() diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -69,6 +69,14 @@ res = interpret(f, []) assert res +def test__encode_states(): + val = rgc._encode_states(42, 43) + assert rgc.old_state(val) == 42 + assert rgc.new_state(val) == 43 + assert not rgc.is_done(val) + # + val = rgc.collect_step() + assert rgc.is_done(val) def test_can_move(): T0 = lltype.GcStruct('T') From pypy.commits at gmail.com Fri Oct 19 12:50:09 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 19 Oct 2018 09:50:09 -0700 (PDT) Subject: [pypy-commit] pypy gc-disable: implement the return value for collect_step in the real incminimark Message-ID: <5bca0b41.1c69fb81.ac4a1.3145@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95223:69e61e0e2314 Date: 2018-10-19 18:43 +0200 http://bitbucket.org/pypy/pypy/changeset/69e61e0e2314/ Log: implement the return value for collect_step in the real incminimark diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -778,14 +778,11 @@ This is meant to be used together with gc.disable(), to have a fine-grained control on when the GC runs. """ - in_progress = self.gc_state != STATE_SCANNING + old_state = self.gc_state self._minor_collection() self.major_collection_step() self.rrc_invoke_callback() - # if we were in the middle of a collection and we are back to - # STATE_SCANNING, it means we have just finished one - done = in_progress and self.gc_state == STATE_SCANNING - return done + return rgc._encode_states(old_state, self.gc_state) def minor_collection_with_major_progress(self, extrasize=0, force_enabled=False): diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py --- a/rpython/memory/gc/test/test_direct.py +++ b/rpython/memory/gc/test/test_direct.py @@ -823,14 +823,24 @@ py.test.raises(RuntimeError, 's.x') def test_collect_step(self, debuglog): + from rpython.rlib import rgc n = 0 + states = [] while True: debuglog.reset() - done = self.gc.collect_step() + val = self.gc.collect_step() + states.append((rgc.old_state(val), rgc.new_state(val))) summary = debuglog.summary() assert summary == {'gc-minor': 1, 'gc-collect-step': 1} - if done: + if rgc.is_done(val): break n += 1 if n == 100: assert False, 'this looks like an endless loop' + # + assert states == [ + (incminimark.STATE_SCANNING, incminimark.STATE_MARKING), + (incminimark.STATE_MARKING, incminimark.STATE_SWEEPING), + (incminimark.STATE_SWEEPING, incminimark.STATE_FINALIZING), + (incminimark.STATE_FINALIZING, incminimark.STATE_SCANNING) + ] diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -310,7 +310,7 @@ self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc, annmodel.SomeInteger()], annmodel.s_None) self.collect_step_ptr = getfn(GCClass.collect_step.im_func, [s_gc], - annmodel.s_Bool) + annmodel.SomeInteger()) self.enable_ptr = getfn(GCClass.enable.im_func, [s_gc], annmodel.s_None) self.disable_ptr = getfn(GCClass.disable.im_func, [s_gc], annmodel.s_None) self.isenabled_ptr = getfn(GCClass.isenabled.im_func, [s_gc], diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1866,9 +1866,12 @@ # gc.disable() n = 0 + states = [] while True: n += 1 - if rgc.collect_step(): + val = rgc.collect_step() + states.append((rgc.old_state(val), rgc.new_state(val))) + if rgc.is_done(val): break if n == 100: print 'Endless loop!' @@ -1877,6 +1880,17 @@ if n < 4: # we expect at least 4 steps print 'Too few steps! n =', n assert False + + # check that the state transitions are reasonable + first_state, _ = states[0] + for i, (old_state, new_state) in enumerate(states): + is_last = (i == len(states) - 1) + is_valid = False + if is_last: + assert old_state != new_state == first_state + else: + assert new_state == old_state or new_state == old_state+1 + return counter.val return f From pypy.commits at gmail.com Sat Oct 20 08:48:46 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 20 Oct 2018 05:48:46 -0700 (PDT) Subject: [pypy-commit] pypy cffi_dlopen_unicode: close branch, ready to merge Message-ID: <5bcb242e.1c69fb81.e49d9.32bd@mx.google.com> Author: Armin Rigo Branch: cffi_dlopen_unicode Changeset: r95224:40f39b2438a8 Date: 2018-10-20 14:01 +0200 http://bitbucket.org/pypy/pypy/changeset/40f39b2438a8/ Log: close branch, ready to merge From pypy.commits at gmail.com Sat Oct 20 08:48:49 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 20 Oct 2018 05:48:49 -0700 (PDT) Subject: [pypy-commit] pypy default: hg merge cffi_dlopen_unicode Message-ID: <5bcb2431.1c69fb81.382c3.afae@mx.google.com> Author: Armin Rigo Branch: Changeset: r95225:6b5c9e6f9782 Date: 2018-10-20 14:01 +0200 http://bitbucket.org/pypy/pypy/changeset/6b5c9e6f9782/ Log: hg merge cffi_dlopen_unicode diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,31 +1,24 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.interpreter.error import oefmt -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from pypy.module._cffi_backend.parse_c_type import ( _CFFI_OPCODE_T, GLOBAL_S, CDL_INTCONST_S, STRUCT_UNION_S, FIELD_S, ENUM_S, TYPENAME_S, ll_set_cdl_realize_global_int) from pypy.module._cffi_backend.realize_c_type import getop from pypy.module._cffi_backend.lib_obj import W_LibObject -from pypy.module._cffi_backend import cffi_opcode, cffi1_module - +from pypy.module._cffi_backend import cffi_opcode, cffi1_module, misc class W_DlOpenLibObject(W_LibObject): - def __init__(self, ffi, filename, flags): - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(ffi.space, e, filename) - W_LibObject.__init__(self, ffi, filename) + def __init__(self, ffi, w_filename, flags): + space = ffi.space + fname, handle = misc.dlopen_w(space, w_filename, flags) + W_LibObject.__init__(self, ffi, fname) self.libhandle = handle - self.register_finalizer(ffi.space) + self.register_finalizer(space) def _finalize_(self): h = self.libhandle diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -572,8 +572,8 @@ return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA) - @unwrap_spec(filename="fsencode_or_none", flags=int) - def descr_dlopen(self, filename, flags=0): + @unwrap_spec(flags=int) + def descr_dlopen(self, w_filename, flags=0): """\ Load and return a dynamic library identified by 'name'. The standard C library can be loaded by passing None. @@ -584,7 +584,7 @@ first access.""" # from pypy.module._cffi_backend import cdlopen - return cdlopen.W_DlOpenLibObject(self, filename, flags) + return cdlopen.W_DlOpenLibObject(self, w_filename, flags) def descr_dlclose(self, w_lib): diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -4,28 +4,21 @@ from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType +from pypy.module._cffi_backend import misc class W_Library(W_Root): _immutable_ = True - def __init__(self, space, filename, flags): + def __init__(self, space, w_filename, flags): self.space = space - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - self.handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, filename) - self.name = filename + self.name, self.handle = misc.dlopen_w(space, w_filename, flags) self.register_finalizer(space) def _finalize_(self): @@ -104,7 +97,7 @@ W_Library.typedef.acceptable_as_base_class = False - at unwrap_spec(filename="fsencode_or_none", flags=int) -def load_library(space, filename, flags=0): - lib = W_Library(space, filename, flags) + at unwrap_spec(flags=int) +def load_library(space, w_filename, flags=0): + lib = W_Library(space, w_filename, flags) return lib diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,14 +1,23 @@ from __future__ import with_statement +import sys from pypy.interpreter.error import OperationError, oefmt +from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rlib import jit from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.rdynload import dlopen, DLOpenError from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + # ____________________________________________________________ @@ -383,3 +392,30 @@ ptr = rffi.cast(rffi.FLOATP, source) for i in range(len(float_list)): float_list[i] = rffi.cast(lltype.Float, ptr[i]) + +# ____________________________________________________________ + +def dlopen_w(space, w_filename, flags): + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.text_w(space.repr(w_filename)) + unicode_name = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(unicode_name) as ll_libname: + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + elif space.isinstance_w(w_filename, space.w_unicode): + fname = space.fsencode_w(w_filename) + else: + fname = space.text_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + return fname, handle diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,8 +1,13 @@ import py +import sys, shutil, os from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache +if sys.platform == 'win32': + WIN32 = True +else: + WIN32 = False class AppTestRecompilerPython: spaceconfig = dict(usemodules=['_cffi_backend']) @@ -40,6 +45,18 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) + if WIN32: + unicode_name = u'load\u03betest.dll' + else: + unicode_name = u'load_caf\xe9' + os.path.splitext(outputfilename)[1] + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None # skip test_dlopen_unicode + if unicode_name is not None: + outputfileUname = os.path.join(unicode(udir), unicode_name) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -108,6 +125,15 @@ assert lib.add42(-10) == 32 assert type(lib.add42) is _cffi_backend.FFI.CData + def test_dlopen_unicode(self): + if not getattr(self, 'extmodU', None): + skip("no unicode file name") + import _cffi_backend + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 + def test_dlclose(self): import _cffi_backend self.fix_path() diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -308,10 +308,10 @@ dldflags = rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY if os.environ.get('CPPYY_BACKEND_LIBRARY'): libname = os.environ['CPPYY_BACKEND_LIBRARY'] - state.backend = W_Library(space, libname, dldflags) + state.backend = W_Library(space, space.newtext(libname), dldflags) else: # try usual lookups - state.backend = W_Library(space, backend_library, dldflags) + state.backend = W_Library(space, space.newtext(backend_library), dldflags) if state.backend: # fix constants diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -242,6 +242,16 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenU(name, mode=-1): + # mode is unused on windows, but a consistant signature + res = rwin32.LoadLibraryW(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: From pypy.commits at gmail.com Sat Oct 20 08:48:51 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 20 Oct 2018 05:48:51 -0700 (PDT) Subject: [pypy-commit] pypy default: simplify Message-ID: <5bcb2433.1c69fb81.fbdb7.02b4@mx.google.com> Author: Armin Rigo Branch: Changeset: r95226:72c9b40c6515 Date: 2018-10-20 14:10 +0200 http://bitbucket.org/pypy/pypy/changeset/72c9b40c6515/ Log: simplify diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -407,10 +407,8 @@ else: if space.is_none(w_filename): fname = None - elif space.isinstance_w(w_filename, space.w_unicode): + else: fname = space.fsencode_w(w_filename) - else: - fname = space.text_w(w_filename) with rffi.scoped_str2charp(fname) as ll_libname: if fname is None: fname = "" From pypy.commits at gmail.com Sat Oct 20 08:48:53 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 20 Oct 2018 05:48:53 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5bcb2435.1c69fb81.7cf08.ce34@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r95227:115a0ddc5b54 Date: 2018-10-20 14:48 +0200 http://bitbucket.org/pypy/pypy/changeset/115a0ddc5b54/ Log: hg merge default diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,31 +1,24 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.interpreter.error import oefmt -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from pypy.module._cffi_backend.parse_c_type import ( _CFFI_OPCODE_T, GLOBAL_S, CDL_INTCONST_S, STRUCT_UNION_S, FIELD_S, ENUM_S, TYPENAME_S, ll_set_cdl_realize_global_int) from pypy.module._cffi_backend.realize_c_type import getop from pypy.module._cffi_backend.lib_obj import W_LibObject -from pypy.module._cffi_backend import cffi_opcode, cffi1_module - +from pypy.module._cffi_backend import cffi_opcode, cffi1_module, misc class W_DlOpenLibObject(W_LibObject): - def __init__(self, ffi, filename, flags): - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(ffi.space, e, filename) - W_LibObject.__init__(self, ffi, filename) + def __init__(self, ffi, w_filename, flags): + space = ffi.space + fname, handle = misc.dlopen_w(space, w_filename, flags) + W_LibObject.__init__(self, ffi, fname) self.libhandle = handle - self.register_finalizer(ffi.space) + self.register_finalizer(space) def _finalize_(self): h = self.libhandle diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -572,8 +572,8 @@ return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA) - @unwrap_spec(filename="fsencode_or_none", flags=int) - def descr_dlopen(self, filename, flags=0): + @unwrap_spec(flags=int) + def descr_dlopen(self, w_filename, flags=0): """\ Load and return a dynamic library identified by 'name'. The standard C library can be loaded by passing None. @@ -584,7 +584,7 @@ first access.""" # from pypy.module._cffi_backend import cdlopen - return cdlopen.W_DlOpenLibObject(self, filename, flags) + return cdlopen.W_DlOpenLibObject(self, w_filename, flags) def descr_dlclose(self, w_lib): diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -4,28 +4,21 @@ from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType +from pypy.module._cffi_backend import misc class W_Library(W_Root): _immutable_ = True - def __init__(self, space, filename, flags): + def __init__(self, space, w_filename, flags): self.space = space - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - self.handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, filename) - self.name = filename + self.name, self.handle = misc.dlopen_w(space, w_filename, flags) self.register_finalizer(space) def _finalize_(self): @@ -104,7 +97,7 @@ W_Library.typedef.acceptable_as_base_class = False - at unwrap_spec(filename="fsencode_or_none", flags=int) -def load_library(space, filename, flags=0): - lib = W_Library(space, filename, flags) + at unwrap_spec(flags=int) +def load_library(space, w_filename, flags=0): + lib = W_Library(space, w_filename, flags) return lib diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,15 +1,24 @@ from __future__ import with_statement +import sys from pypy.interpreter.error import OperationError, oefmt +from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rlib import jit from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.rdynload import dlopen, DLOpenError from rpython.rlib.nonconst import NonConstant from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + # ____________________________________________________________ @@ -393,3 +402,28 @@ ptr = rffi.cast(rffi.FLOATP, source) for i in range(len(float_list)): float_list[i] = rffi.cast(lltype.Float, ptr[i]) + +# ____________________________________________________________ + +def dlopen_w(space, w_filename, flags): + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.text_w(space.repr(w_filename)) + unicode_name = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(unicode_name) as ll_libname: + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + else: + fname = space.fsencode_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + return fname, handle diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,8 +1,13 @@ import py +import sys, shutil, os from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache +if sys.platform == 'win32': + WIN32 = True +else: + WIN32 = False class AppTestRecompilerPython: spaceconfig = dict(usemodules=['_cffi_backend']) @@ -40,6 +45,18 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) + if WIN32: + unicode_name = u'load\u03betest.dll' + else: + unicode_name = u'load_caf\xe9' + os.path.splitext(outputfilename)[1] + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None # skip test_dlopen_unicode + if unicode_name is not None: + outputfileUname = os.path.join(unicode(udir), unicode_name) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -108,6 +125,16 @@ assert lib.add42(-10) == 32 assert type(lib.add42) is _cffi_backend.FFI.CData + def test_dlopen_unicode(self): + if not getattr(self, 'extmodU', None): + skip("no unicode file name") + import _cffi_backend, sys + sys.pypy_initfsencoding() # initialize space.sys.filesystemencoding + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 + def test_dlclose(self): import _cffi_backend self.fix_path() diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -308,10 +308,10 @@ dldflags = rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY if os.environ.get('CPPYY_BACKEND_LIBRARY'): libname = os.environ['CPPYY_BACKEND_LIBRARY'] - state.backend = W_Library(space, libname, dldflags) + state.backend = W_Library(space, space.newtext(libname), dldflags) else: # try usual lookups - state.backend = W_Library(space, backend_library, dldflags) + state.backend = W_Library(space, space.newtext(backend_library), dldflags) if state.backend: # fix constants diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -242,6 +242,16 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenU(name, mode=-1): + # mode is unused on windows, but a consistant signature + res = rwin32.LoadLibraryW(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: From pypy.commits at gmail.com Sun Oct 21 14:40:32 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 21 Oct 2018 11:40:32 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: typo Message-ID: <5bccc820.1c69fb81.38190.f1ca@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95228:10fdcbcd9d3f Date: 2018-10-21 21:39 +0300 http://bitbucket.org/pypy/pypy/changeset/10fdcbcd9d3f/ Log: typo diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -108,7 +108,7 @@ from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) if _WIN32: - errorhandler=state.encode_error_handler, + errorhandler=state.encode_error_handler utf8 = space.utf8_w(w_uni) bytes = utf8_encode_mbcs(utf8, 'strict', errorhandler) elif _MACOSX: From pypy.commits at gmail.com Sun Oct 21 14:40:35 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 21 Oct 2018 11:40:35 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5bccc823.1c69fb81.b490b.69ba@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95229:403463fc684e Date: 2018-10-21 21:39 +0300 http://bitbucket.org/pypy/pypy/changeset/403463fc684e/ Log: merge py3.5 into branch diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py --- a/lib-python/3/distutils/sysconfig_pypy.py +++ b/lib-python/3/distutils/sysconfig_pypy.py @@ -8,11 +8,9 @@ available. """ -__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $" - import sys import os -import imp +import imp, _imp from distutils.errors import DistutilsPlatformError @@ -71,9 +69,17 @@ def _init_nt(): """Initialize the module as appropriate for NT""" g = {} + # set basic install directories + g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) + g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) + + # XXX hmmm.. a normal install puts include files here + g['INCLUDEPY'] = get_python_inc(plat_specific=0) + + g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] g['EXE'] = ".exe" - g['SO'] = ".pyd" - g['SOABI'] = g['SO'].rsplit('.')[0] # xxx? + g['VERSION'] = get_python_version().replace(".", "") + g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) global _config_vars _config_vars = g diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -1,31 +1,24 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.interpreter.error import oefmt -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from pypy.module._cffi_backend.parse_c_type import ( _CFFI_OPCODE_T, GLOBAL_S, CDL_INTCONST_S, STRUCT_UNION_S, FIELD_S, ENUM_S, TYPENAME_S, ll_set_cdl_realize_global_int) from pypy.module._cffi_backend.realize_c_type import getop from pypy.module._cffi_backend.lib_obj import W_LibObject -from pypy.module._cffi_backend import cffi_opcode, cffi1_module - +from pypy.module._cffi_backend import cffi_opcode, cffi1_module, misc class W_DlOpenLibObject(W_LibObject): - def __init__(self, ffi, filename, flags): - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(ffi.space, e, filename) - W_LibObject.__init__(self, ffi, filename) + def __init__(self, ffi, w_filename, flags): + space = ffi.space + fname, handle = misc.dlopen_w(space, w_filename, flags) + W_LibObject.__init__(self, ffi, fname) self.libhandle = handle - self.register_finalizer(ffi.space) + self.register_finalizer(space) def _finalize_(self): h = self.libhandle diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -572,8 +572,8 @@ return self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CDATA) - @unwrap_spec(filename="fsencode_or_none", flags=int) - def descr_dlopen(self, filename, flags=0): + @unwrap_spec(flags=int) + def descr_dlopen(self, w_filename, flags=0): """\ Load and return a dynamic library identified by 'name'. The standard C library can be loaded by passing None. @@ -584,7 +584,7 @@ first access.""" # from pypy.module._cffi_backend import cdlopen - return cdlopen.W_DlOpenLibObject(self, filename, flags) + return cdlopen.W_DlOpenLibObject(self, w_filename, flags) def descr_dlclose(self, w_lib): diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -4,28 +4,21 @@ from pypy.interpreter.error import oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rtyper.lltypesystem import rffi -from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rlib.rdynload import DLLHANDLE, dlsym, dlclose from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType +from pypy.module._cffi_backend import misc class W_Library(W_Root): _immutable_ = True - def __init__(self, space, filename, flags): + def __init__(self, space, w_filename, flags): self.space = space - with rffi.scoped_str2charp(filename) as ll_libname: - if filename is None: - filename = "" - try: - self.handle = dlopen(ll_libname, flags) - except DLOpenError as e: - raise wrap_dlopenerror(space, e, filename) - self.name = filename + self.name, self.handle = misc.dlopen_w(space, w_filename, flags) self.register_finalizer(space) def _finalize_(self): @@ -104,7 +97,7 @@ W_Library.typedef.acceptable_as_base_class = False - at unwrap_spec(filename="fsencode_or_none", flags=int) -def load_library(space, filename, flags=0): - lib = W_Library(space, filename, flags) + at unwrap_spec(flags=int) +def load_library(space, w_filename, flags=0): + lib = W_Library(space, w_filename, flags) return lib diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,15 +1,24 @@ from __future__ import with_statement +import sys from pypy.interpreter.error import OperationError, oefmt +from pypy.module._rawffi.interp_rawffi import wrap_dlopenerror from rpython.rlib import jit from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rarithmetic import r_uint, r_ulonglong from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.rdynload import dlopen, DLOpenError from rpython.rlib.nonconst import NonConstant from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +if sys.platform == 'win32': + from rpython.rlib.rdynload import dlopenU + WIN32 = True +else: + WIN32 = False + # ____________________________________________________________ @@ -393,3 +402,28 @@ ptr = rffi.cast(rffi.FLOATP, source) for i in range(len(float_list)): float_list[i] = rffi.cast(lltype.Float, ptr[i]) + +# ____________________________________________________________ + +def dlopen_w(space, w_filename, flags): + if WIN32 and space.isinstance_w(w_filename, space.w_unicode): + fname = space.text_w(space.repr(w_filename)) + unicode_name = space.unicode_w(w_filename) + with rffi.scoped_unicode2wcharp(unicode_name) as ll_libname: + try: + handle = dlopenU(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + else: + if space.is_none(w_filename): + fname = None + else: + fname = space.fsencode_w(w_filename) + with rffi.scoped_str2charp(fname) as ll_libname: + if fname is None: + fname = "" + try: + handle = dlopen(ll_libname, flags) + except DLOpenError as e: + raise wrap_dlopenerror(space, e, fname) + return fname, handle diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -1,8 +1,13 @@ import py +import sys, shutil, os from rpython.tool.udir import udir from pypy.interpreter.gateway import interp2app from pypy.module._cffi_backend.newtype import _clean_cache +if sys.platform == 'win32': + WIN32 = True +else: + WIN32 = False class AppTestRecompilerPython: spaceconfig = dict(usemodules=['_cffi_backend']) @@ -40,6 +45,18 @@ 'globalconst42', 'globalconsthello']) outputfilename = ffiplatform.compile(str(tmpdir), ext) cls.w_extmod = space.wrap(outputfilename) + if WIN32: + unicode_name = u'load\u03betest.dll' + else: + unicode_name = u'load_caf\xe9' + os.path.splitext(outputfilename)[1] + try: + unicode_name.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + unicode_name = None # skip test_dlopen_unicode + if unicode_name is not None: + outputfileUname = os.path.join(unicode(udir), unicode_name) + shutil.copyfile(outputfilename, outputfileUname) + cls.w_extmodU = space.wrap(outputfileUname) #mod.tmpdir = tmpdir # ffi = FFI() @@ -108,6 +125,16 @@ assert lib.add42(-10) == 32 assert type(lib.add42) is _cffi_backend.FFI.CData + def test_dlopen_unicode(self): + if not getattr(self, 'extmodU', None): + skip("no unicode file name") + import _cffi_backend, sys + sys.pypy_initfsencoding() # initialize space.sys.filesystemencoding + self.fix_path() + from re_python_pysrc import ffi + lib = ffi.dlopen(self.extmodU) + assert lib.add42(-10) == 32 + def test_dlclose(self): import _cffi_backend self.fix_path() diff --git a/pypy/module/_cppyy/capi/loadable_capi.py b/pypy/module/_cppyy/capi/loadable_capi.py --- a/pypy/module/_cppyy/capi/loadable_capi.py +++ b/pypy/module/_cppyy/capi/loadable_capi.py @@ -308,10 +308,10 @@ dldflags = rdynload.RTLD_LOCAL | rdynload.RTLD_LAZY if os.environ.get('CPPYY_BACKEND_LIBRARY'): libname = os.environ['CPPYY_BACKEND_LIBRARY'] - state.backend = W_Library(space, libname, dldflags) + state.backend = W_Library(space, space.newtext(libname), dldflags) else: # try usual lookups - state.backend = W_Library(space, backend_library, dldflags) + state.backend = W_Library(space, space.newtext(backend_library), dldflags) if state.backend: # fix constants diff --git a/rpython/rlib/rdynload.py b/rpython/rlib/rdynload.py --- a/rpython/rlib/rdynload.py +++ b/rpython/rlib/rdynload.py @@ -242,6 +242,16 @@ raise DLOpenError(ustr.encode('utf-8')) return res + def dlopenU(name, mode=-1): + # mode is unused on windows, but a consistant signature + res = rwin32.LoadLibraryW(name) + if not res: + err = rwin32.GetLastError_saved() + ustr = rwin32.FormatErrorW(err) + # DLOpenError unicode msg breaks translation of cpyext create_extension_module + raise DLOpenError(ustr.encode('utf-8')) + return res + def dlclose(handle): res = rwin32.FreeLibrary(handle) if res: diff --git a/testrunner/lib_python_tests.py b/testrunner/lib_python_tests.py --- a/testrunner/lib_python_tests.py +++ b/testrunner/lib_python_tests.py @@ -21,6 +21,7 @@ pypyopt, "--timeout=3600", "-rs", + "--duration=10", "--resultlog=cpython.log", "lib-python", ] + sys.argv[1:], cwd=rootdir) From pypy.commits at gmail.com Sun Oct 21 15:50:01 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 21 Oct 2018 12:50:01 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: use utf8 in further encoding if error handler "fixes" surrogates Message-ID: <5bccd869.1c69fb81.697d.4057@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95230:9f83d1c579c4 Date: 2018-10-21 22:49 +0300 http://bitbucket.org/pypy/pypy/changeset/9f83d1c579c4/ Log: use utf8 in further encoding if error handler "fixes" surrogates diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1201,11 +1201,11 @@ def encode_object(space, w_object, encoding, errors, allow_surrogates=False): - utf8 = space.utf8_w(w_object) # TODO: refactor unnatrual use of error hanlders here, # we should make a single pass over the utf8 str from pypy.module._codecs.interp_codecs import encode_text, CodecState if not allow_surrogates: + utf8 = space.utf8_w(w_object) if errors is None: errors = 'strict' pos = rutf8.surrogate_in_utf8(utf8) @@ -1215,9 +1215,12 @@ start = utf8[:pos] ru, pos = eh(errors, "utf8", "surrogates not allowed", utf8, pos, pos + 1) - end = utf8[pos+1:] + upos = rutf8.next_codepoint_pos(utf8,pos) + end = utf8[upos+1:] utf8 = start + ru + end + w_object = space.newtext(utf8) if errors is None or errors == 'strict': + utf8 = space.utf8_w(w_object) if encoding is None or encoding == 'utf-8': #if rutf8.has_surrogates(utf8): # utf8 = rutf8.reencode_utf8_with_surrogates(utf8) From pypy.commits at gmail.com Mon Oct 22 02:00:23 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 21 Oct 2018 23:00:23 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: off-by-one error Message-ID: <5bcd6777.1c69fb81.1a261.f9a3@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95231:8c089ffd30f2 Date: 2018-10-22 08:49 +0300 http://bitbucket.org/pypy/pypy/changeset/8c089ffd30f2/ Log: off-by-one error diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1216,7 +1216,7 @@ ru, pos = eh(errors, "utf8", "surrogates not allowed", utf8, pos, pos + 1) upos = rutf8.next_codepoint_pos(utf8,pos) - end = utf8[upos+1:] + end = utf8[upos:] utf8 = start + ru + end w_object = space.newtext(utf8) if errors is None or errors == 'strict': From pypy.commits at gmail.com Wed Oct 24 00:23:27 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 23 Oct 2018 21:23:27 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: fix for MAXUNICODE < 65536 Message-ID: <5bcff3bf.1c69fb81.92fdb.ad8a@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95233:26082fc25722 Date: 2018-10-24 07:22 +0300 http://bitbucket.org/pypy/pypy/changeset/26082fc25722/ Log: fix for MAXUNICODE < 65536 diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1530,7 +1530,10 @@ if size == 0: return '', 0 - unicode_bytes = 4 + if runicode.MAXUNICODE < 65536: + unicode_bytes = 2 + else: + unicode_bytes = 4 if BYTEORDER == "little": start = 0 stop = unicode_bytes @@ -1554,7 +1557,7 @@ for j in range(start, stop, step): t += r_uint(ord(s[pos + j])) << (h*8) h += 1 - if t > 0x10ffff: + if t > runicode.MAXUNICODE: res, pos = errorhandler(errors, "unicode_internal", "unichr(%d) not in range" % (t,), s, pos, pos + unicode_bytes) @@ -1571,18 +1574,24 @@ if size == 0: return '' - result = StringBuilder(size * 4) + if runicode.MAXUNICODE < 65536: + unicode_bytes = 2 + else: + unicode_bytes = 4 + result = StringBuilder(size * unicode_bytes) pos = 0 while pos < size: oc = rutf8.codepoint_at_pos(s, pos) if BYTEORDER == "little": result.append(chr(oc & 0xFF)) result.append(chr(oc >> 8 & 0xFF)) - result.append(chr(oc >> 16 & 0xFF)) - result.append(chr(oc >> 24 & 0xFF)) + if unicode_bytes > 2: + result.append(chr(oc >> 16 & 0xFF)) + result.append(chr(oc >> 24 & 0xFF)) else: - result.append(chr(oc >> 24 & 0xFF)) - result.append(chr(oc >> 16 & 0xFF)) + if unicode_bytes > 2: + result.append(chr(oc >> 24 & 0xFF)) + result.append(chr(oc >> 16 & 0xFF)) result.append(chr(oc >> 8 & 0xFF)) result.append(chr(oc & 0xFF)) pos = rutf8.next_codepoint_pos(s, pos) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -248,7 +248,7 @@ check_exception(space, w_exc) if space.isinstance_w(w_exc, space.w_UnicodeEncodeError): w_obj = space.getattr(w_exc, space.newtext('object')) - space.realutf8_w(w_obj) # weeoes + space.realutf8_w(w_obj) # for errors w_obj = space.convert_arg_to_w_unicode(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) @@ -275,17 +275,22 @@ check_exception(space, w_exc) if (space.isinstance_w(w_exc, space.w_UnicodeEncodeError) or - space.isinstance_w(w_exc, space.w_UnicodeTranslateError)): - obj = space.realunicode_w(space.getattr(w_exc, space.newtext('object'))) + space.isinstance_w(w_exc, space.w_UnicodeTranslateError)): + w_obj = space.getattr(w_exc, space.newtext('object')) + space.realutf8_w(w_obj) # for errors + w_obj = space.convert_arg_to_w_unicode(w_obj) start = space.int_w(space.getattr(w_exc, space.newtext('start'))) w_end = space.getattr(w_exc, space.newtext('end')) end = space.int_w(w_end) + start = w_obj._index_to_byte(start) + end = w_obj._index_to_byte(end) builder = StringBuilder() pos = start + obj = w_obj._utf8 while pos < end: - oc = ord(obj[pos]) - raw_unicode_escape_helper(builder, oc) - pos += 1 + code = rutf8.codepoint_at_pos(obj, pos) + raw_unicode_escape_helper(builder, code) + pos = rutf8.next_codepoint_pos(obj, pos) return space.newtuple([space.newtext(builder.build()), w_end]) elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError): obj = space.bytes_w(space.getattr(w_exc, space.newtext('object'))) From pypy.commits at gmail.com Wed Oct 24 00:23:29 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 23 Oct 2018 21:23:29 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: for py3, MAXUNICODE always 0x10ffff Message-ID: <5bcff3c1.1c69fb81.cb368.bf06@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95234:a6872f6289d6 Date: 2018-10-24 07:22 +0300 http://bitbucket.org/pypy/pypy/changeset/a6872f6289d6/ Log: for py3, MAXUNICODE always 0x10ffff diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -8,7 +8,7 @@ from rpython.rlib import jit, nonconst -if rffi.sizeof(lltype.UniChar) == 4: +if 1 or rffi.sizeof(lltype.UniChar) == 4: MAXUNICODE = 0x10ffff allow_surrogate_by_default = False else: From pypy.commits at gmail.com Wed Oct 24 12:14:47 2018 From: pypy.commits at gmail.com (antocuni) Date: Wed, 24 Oct 2018 09:14:47 -0700 (PDT) Subject: [pypy-commit] pypy gc-disable: improve the return value of gc.collect_step: now it returns a GcCollectStepStats, the same which is passed to the on_gc_collect_step hook; in particular, now we can know the old and the new GC state Message-ID: <5bd09a77.1c69fb81.38080.74fd@mx.google.com> Author: Antonio Cuni Branch: gc-disable Changeset: r95235:2db479ad4b70 Date: 2018-10-24 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/2db479ad4b70/ Log: improve the return value of gc.collect_step: now it returns a GcCollectStepStats, the same which is passed to the on_gc_collect_step hook; in particular, now we can know the old and the new GC state diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py --- a/pypy/module/gc/hook.py +++ b/pypy/module/gc/hook.py @@ -1,5 +1,6 @@ from rpython.memory.gc.hook import GcHooks -from rpython.memory.gc import incminimark +from rpython.memory.gc import incminimark +from rpython.rlib import rgc from rpython.rlib.nonconst import NonConstant from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault @@ -190,7 +191,8 @@ self.duration_min, self.duration_max, self.oldstate, - self.newstate) + self.newstate, + rgc.is_done__states(self.oldstate, self.newstate)) self.reset() self.space.call_function(self.w_callable, w_stats) @@ -250,15 +252,32 @@ class W_GcCollectStepStats(W_Root): + # NOTE: this is specific to incminimark: if we want to integrate the + # applevel gc module with another gc, we probably need a more general + # approach to this. + # + # incminimark has 4 GC states: scanning, marking, sweeping and + # finalizing. However, from the user point of view, we have an additional + # "virtual" state: USERDEL, which represent when we run applevel + # finalizers after having completed a GC major collection. This state is + # never explicitly visible when using hooks, but it is used for the return + # value of gc.collect_step (see interp_gc.py) + STATE_SCANNING = incminimark.STATE_SCANNING + STATE_MARKING = incminimark.STATE_MARKING + STATE_SWEEPING = incminimark.STATE_SWEEPING + STATE_FINALIZING = incminimark.STATE_FINALIZING + STATE_USERDEL = incminimark.STATE_FINALIZING + 1 # used by StepCollector + GC_STATES = tuple(incminimark.GC_STATES + ['USERDEL']) def __init__(self, count, duration, duration_min, duration_max, - oldstate, newstate): + oldstate, newstate, major_is_done): self.count = count self.duration = duration self.duration_min = duration_min self.duration_max = duration_max self.oldstate = oldstate self.newstate = newstate + self.major_is_done = major_is_done class W_GcCollectStats(W_Root): @@ -314,11 +333,16 @@ W_GcCollectStepStats.typedef = TypeDef( "GcCollectStepStats", - STATE_SCANNING = incminimark.STATE_SCANNING, - STATE_MARKING = incminimark.STATE_MARKING, - STATE_SWEEPING = incminimark.STATE_SWEEPING, - STATE_FINALIZING = incminimark.STATE_FINALIZING, - GC_STATES = tuple(incminimark.GC_STATES), + STATE_SCANNING = W_GcCollectStepStats.STATE_SCANNING, + STATE_MARKING = W_GcCollectStepStats.STATE_MARKING, + STATE_SWEEPING = W_GcCollectStepStats.STATE_SWEEPING, + STATE_FINALIZING = W_GcCollectStepStats.STATE_FINALIZING, + STATE_USERDEL = W_GcCollectStepStats.STATE_USERDEL, + GC_STATES = tuple(W_GcCollectStepStats.GC_STATES), + major_is_done = interp_attrproperty( + "major_is_done", + cls=W_GcCollectStepStats, + wrapfn="newbool"), **wrap_many_ints(W_GcCollectStepStats, ( "count", "duration", diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py --- a/pypy/module/gc/interp_gc.py +++ b/pypy/module/gc/interp_gc.py @@ -1,6 +1,7 @@ from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import oefmt from rpython.rlib import rgc +from pypy.module.gc.hook import W_GcCollectStepStats @unwrap_spec(generation=int) @@ -96,12 +97,27 @@ if self.finalizing: self._run_finalizers() self.finalizing = False - return True # everything done + oldstate = W_GcCollectStepStats.STATE_USERDEL + newstate = W_GcCollectStepStats.STATE_SCANNING + major_is_done = True # now we are finally done else: - done = self._collect_step() - if done: + states = self._collect_step() + oldstate = rgc.old_state(states) + newstate = rgc.new_state(states) + major_is_done = False # USERDEL still to do + if rgc.is_done(states): + newstate = W_GcCollectStepStats.STATE_USERDEL self.finalizing = True - return False # still something to do + # + duration = -1 + return W_GcCollectStepStats( + count = 1, + duration = duration, + duration_min = duration, + duration_max = duration, + oldstate = oldstate, + newstate = newstate, + major_is_done = major_is_done) def _collect_step(self): return rgc.collect_step() @@ -116,7 +132,8 @@ If the GC is not incremental, do a full collection and return True. """ sc = space.fromcache(StepCollector) - return space.newbool(sc.do()) + w_stats = sc.do() + return w_stats # ____________________________________________________________ diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py --- a/pypy/module/gc/test/test_gc.py +++ b/pypy/module/gc/test/test_gc.py @@ -3,7 +3,7 @@ from rpython.rlib import rgc from pypy.interpreter.baseobjspace import ObjSpace from pypy.interpreter.gateway import interp2app, unwrap_spec -from pypy.module.gc.interp_gc import StepCollector +from pypy.module.gc.interp_gc import StepCollector, W_GcCollectStepStats class AppTestGC(object): @@ -113,7 +113,7 @@ n = 0 while True: n += 1 - if gc.collect_step(): + if gc.collect_step().major_is_done: break assert n >= 2 # at least one step + 1 finalizing @@ -194,33 +194,52 @@ def test_StepCollector(): + W = W_GcCollectStepStats + SCANNING = W.STATE_SCANNING + MARKING = W.STATE_MARKING + SWEEPING = W.STATE_SWEEPING + FINALIZING = W.STATE_FINALIZING + USERDEL = W.STATE_USERDEL + class MyStepCollector(StepCollector): my_steps = 0 my_done = False my_finalized = 0 + def __init__(self): + StepCollector.__init__(self, space=None) + self._state_transitions = iter([ + (SCANNING, MARKING), + (MARKING, SWEEPING), + (SWEEPING, FINALIZING), + (FINALIZING, SCANNING)]) + def _collect_step(self): self.my_steps += 1 - return self.my_done + try: + oldstate, newstate = next(self._state_transitions) + except StopIteration: + assert False, 'should not happen, did you call _collect_step too much?' + return rgc._encode_states(oldstate, newstate) def _run_finalizers(self): self.my_finalized += 1 - sc = MyStepCollector(space=None) - assert not sc.do() - assert sc.my_steps == 1 - assert not sc.do() - assert sc.my_steps == 2 - sc.my_done = True - assert not sc.do() - assert sc.my_steps == 3 - assert sc.my_finalized == 0 - assert sc.finalizing - assert sc.do() - assert sc.my_steps == 3 - assert sc.my_finalized == 1 - assert not sc.finalizing - assert not sc.do() - assert sc.my_steps == 4 - assert sc.my_finalized == 1 + sc = MyStepCollector() + transitions = [] + while True: + result = sc.do() + transitions.append((result.oldstate, result.newstate, sc.my_finalized)) + if result.major_is_done: + break + assert transitions == [ + (SCANNING, MARKING, False), + (MARKING, SWEEPING, False), + (SWEEPING, FINALIZING, False), + (FINALIZING, USERDEL, False), + (USERDEL, SCANNING, True) + ] + # there is one more transition than actual step, because + # FINALIZING->USERDEL is "virtual" + assert sc.my_steps == len(transitions) - 1 diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py --- a/pypy/module/gc/test/test_hook.py +++ b/pypy/module/gc/test/test_hook.py @@ -69,26 +69,29 @@ def test_on_gc_collect_step(self): import gc + SCANNING = 0 + MARKING = 1 + SWEEPING = 2 + FINALIZING = 3 lst = [] def on_gc_collect_step(stats): lst.append((stats.count, stats.duration, stats.oldstate, - stats.newstate)) + stats.newstate, + stats.major_is_done)) gc.hooks.on_gc_collect_step = on_gc_collect_step - self.fire_gc_collect_step(10, 20, 30) - self.fire_gc_collect_step(40, 50, 60) + self.fire_gc_collect_step(10, SCANNING, MARKING) + self.fire_gc_collect_step(40, FINALIZING, SCANNING) assert lst == [ - (1, 10, 20, 30), - (1, 40, 50, 60), + (1, 10, SCANNING, MARKING, False), + (1, 40, FINALIZING, SCANNING, True), ] # gc.hooks.on_gc_collect_step = None - self.fire_gc_collect_step(70, 80, 90) # won't fire - assert lst == [ - (1, 10, 20, 30), - (1, 40, 50, 60), - ] + oldlst = lst[:] + self.fire_gc_collect_step(70, SCANNING, MARKING) # won't fire + assert lst == oldlst def test_on_gc_collect(self): import gc @@ -123,7 +126,8 @@ assert S.STATE_MARKING == 1 assert S.STATE_SWEEPING == 2 assert S.STATE_FINALIZING == 3 - assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING') + assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', + 'FINALIZING', 'USERDEL') def test_cumulative(self): import gc diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -33,20 +33,30 @@ def _encode_states(oldstate, newstate): return oldstate << 16 | newstate -def old_state(val): - return (val & 0xFFFF0000) >> 16 +def old_state(states): + return (states & 0xFFFF0000) >> 16 -def new_state(val): - return val & 0xFFFF +def new_state(states): + return states & 0xFFFF -def is_done(val): +def is_done(states): + """ + Return True if the return value of collect_step signals the end of a major + collection + """ + old = old_state(states) + new = new_state(states) + return is_done__states(old, new) + +def is_done__states(oldstate, newstate): + "Like is_done, but takes oldstate and newstate explicitly" # a collection is considered done when it ends up in the starting state # (which is usually represented as 0). This logic works for incminimark, # which is currently the only gc actually used and for which collect_step # is implemented. In case we add more GC in the future, we might want to # delegate this logic to the GC itself, but for now it is MUCH simpler to # just write it in plain RPython. - return old_state(val) != 0 and new_state(val) == 0 + return oldstate != 0 and newstate == 0 def set_max_heap_size(nbytes): """Limit the heap size to n bytes. From pypy.commits at gmail.com Thu Oct 25 05:31:54 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 25 Oct 2018 02:31:54 -0700 (PDT) Subject: [pypy-commit] pypy default: Remove bogus (but unused) timeout argument Message-ID: <5bd18d8a.1c69fb81.1cbbc.2fd0@mx.google.com> Author: Armin Rigo Branch: Changeset: r95239:7e0de1ef03b4 Date: 2018-10-25 11:31 +0200 http://bitbucket.org/pypy/pypy/changeset/7e0de1ef03b4/ Log: Remove bogus (but unused) timeout argument diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -70,8 +70,8 @@ pass def register(self, fd, flag): self.fd = fd - def poll(self, timeout=None): - r,w,e = select.select([self.fd],[],[],timeout) + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r,w,e = select.select([self.fd],[],[]) return r POLLIN = getattr(select, "POLLIN", None) From pypy.commits at gmail.com Thu Oct 25 08:19:47 2018 From: pypy.commits at gmail.com (stian) Date: Thu, 25 Oct 2018 05:19:47 -0700 (PDT) Subject: [pypy-commit] pypy math-improvements: Merge default Message-ID: <5bd1b4e3.1c69fb81.5672f.8320@mx.google.com> Author: stian Branch: math-improvements Changeset: r95240:8679952ae1fd Date: 2018-10-25 13:55 +0200 http://bitbucket.org/pypy/pypy/changeset/8679952ae1fd/ Log: Merge default diff too long, truncating to 2000 out of 44960 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -33,7 +33,12 @@ 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1 +4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1 +4909c06daf41ce88f87dc01c57959cadad4df4a8 RevDB-pypy2.7-v5.4.1 +d7724c0a5700b895a47de44074cdf5fd659a988f RevDB-pypy2.7-v5.4.1 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0 +e90317857d27917bf840caf675832292ee070510 RevDB-pypy2.7-v5.6.1 +a24d6c7000c8099c73d3660857f7e3cee5ac045c RevDB-pypy2.7-v5.6.2 fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1 @@ -51,3 +56,5 @@ 0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 +ab0b9caf307db6592905a80b8faffd69b39005b8 release-pypy2.7-v6.0.0 +fdd60ed87e941677e8ea11acf9f1819466521bf2 release-pypy3.5-v6.0.0 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -6,36 +6,36 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', -'py', and '_pytest' directories is licensed as follows: +'py', and '_pytest' directories is licensed as follows: The MIT License - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2018 ------------------------------------ +-------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each -copyrighted by one or more of the following people and organizations: +copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski @@ -89,13 +89,13 @@ Niko Matsakis Alexander Hesse Ludovic Aubry + stian Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski - stian Jan de Mooij Tyler Wade Vincent Legoll @@ -123,10 +123,10 @@ Wenzhu Man Konstantin Lopuhin John Witulski + Jeremy Thurgood Greg Price Ivan Sichmann Freitas Dario Bertini - Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape @@ -145,18 +145,19 @@ Adrian Kuhn tav Georg Brandl + Joannah Nanjekye Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume - Joannah Nanjekye Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller + Dodan Mihai Tim Felgentreff Eugene Oden - Dodan Mihai + Colin Valliant Jeff Terrace Henry Mason Vasily Kuznetsov @@ -225,12 +226,14 @@ Vaibhav Sood Reuben Cummings Attila Gobi + Floris Bruynooghe Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa + Arianna Avanzini Jens-Uwe Mager Valentina Mukhamedzhanova Stefano Parmesan @@ -244,15 +247,18 @@ Lukas Vacek Omer Katz Jacek Generowicz + Tomasz Dziopa Sylvain Thenault Jakub Stasiak Andrew Dalke Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor + Barry Hart Travis Francis Athougies Yasir Suhail Sergey Kishchenko @@ -260,6 +266,7 @@ Lutz Paelike Ian Foote Philipp Rustemeuer + Logan Chien Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez @@ -295,19 +302,20 @@ Akira Li Gustavo Niemeyer Rafał Gałczyński - Logan Chien Lucas Stadler roberto at goyle Matt Bogosian Yury V. Zaytsev florinpapa Anders Sigfridsson + Matt Jackson Nikolay Zinov rafalgalczynski at gmail.com Joshua Gilbert Anna Katrina Dominguez Kim Jin Su Amber Brown + Miro Hrončok Anthony Sottile Nate Bragg Ben Darnell @@ -315,7 +323,6 @@ Godefroid Chappelle Julian Berman Michael Hudson-Doyle - Floris Bruynooghe Stephan Busemann Dan Colish timo @@ -357,6 +364,7 @@ Michael Chermside Anna Ravencroft remarkablerocket + Pauli Virtanen Petre Vijiac Berker Peksag Christian Muirhead @@ -381,6 +389,7 @@ Graham Markall Dan Loewenherz werat + Andrew Stepanov Niclas Olofsson Chris Pressey Tobias Diaz @@ -395,14 +404,14 @@ m at funkyhat.org Stefan Marr - Heinrich-Heine University, Germany + Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden - merlinux GmbH, Germany - tismerysoft GmbH, Germany - Logilab Paris, France - DFKI GmbH, Germany + merlinux GmbH, Germany + tismerysoft GmbH, Germany + Logilab Paris, France + DFKI GmbH, Germany Impara, Germany - Change Maker, Sweden + Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London @@ -410,14 +419,14 @@ The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. - -License for 'lib-python/2.7' -============================ + +License for 'lib-python/2.7, lib-python/3' +========================================== Except when otherwise stated (look for LICENSE files or copyright/license -information at the beginning of each file) the files in the 'lib-python/2.7' +information at the beginning of each file) the files in the 'lib-python' directory are all copyrighted by the Python Software Foundation and licensed -under the terms that you can find here: https://docs.python.org/2/license.html +under the terms that you can find here: https://docs.python.org/3/license.html License for 'pypy/module/unicodedata/' ====================================== @@ -441,9 +450,9 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,42 +4,40 @@ Welcome to PyPy! -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. +PyPy is an interpreter that implements the Python programming language, based +on the RPython compiler framework for dynamic language implementations. -The home page is: +The home page for the interpreter is: http://pypy.org/ -If you want to help developing PyPy, this document might help you: +If you want to help developing PyPy, this documentation might help you: http://doc.pypy.org/ -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! +More documentation about the RPython framework can be found here: + + http://rpython.readthedocs.io/ + +The source for the documentation is in the pypy/doc directory. + + +Using PyPy instead of CPython +----------------------------- + +Please read the information at http://pypy.org/ to find the correct way to +download and use PyPy as an alternative to CPython. + + +Building +-------- + +Building PyPy is not the recommended way to obtain the PyPy alternative python +interpreter. It is time-consuming and requires significant computing resources. +More information can be found here: + + http://doc.pypy.org/en/latest/build.html + +Enjoy and send us feedback! the pypy-dev team - - -Building -======== - -First switch to or download the correct branch. The basic choices are -``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X -branch (e.g. ``py3.5``). - -Build with: - -.. code-block:: console - - $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py - -This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy -directory. We suggest to use virtualenv with the resulting -pypy-c/pypy3-c as the interpreter; you can find more details about -various installation schemes here: - - http://doc.pypy.org/en/latest/install.html diff --git a/dotviewer/font/NOTICE b/dotviewer/font/COPYING.txt rename from dotviewer/font/NOTICE rename to dotviewer/font/COPYING.txt diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py rename from pypy/module/test_lib_pypy/pyrepl/__init__.py rename to extra_tests/test_pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/extra_tests/test_pyrepl/__init__.py @@ -1,3 +1,1 @@ -import sys -import lib_pypy.pyrepl -sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py rename to extra_tests/test_pyrepl/infrastructure.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py rename to extra_tests/test_pyrepl/test_basic.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py rename to extra_tests/test_pyrepl/test_bugs.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py rename from pypy/module/test_lib_pypy/pyrepl/test_functional.py rename to extra_tests/test_pyrepl/test_functional.py --- a/pypy/module/test_lib_pypy/pyrepl/test_functional.py +++ b/extra_tests/test_pyrepl/test_functional.py @@ -7,7 +7,8 @@ import sys -def pytest_funcarg__child(request): + at pytest.fixture() +def child(): try: import pexpect except ImportError: diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py rename to extra_tests/test_pyrepl/test_keymap.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py rename to extra_tests/test_pyrepl/test_reader.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py rename to extra_tests/test_pyrepl/test_readline.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py rename to extra_tests/test_pyrepl/test_wishes.py diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/lib-python/2.7/code.py b/lib-python/2.7/code.py --- a/lib-python/2.7/code.py +++ b/lib-python/2.7/code.py @@ -104,6 +104,12 @@ except SystemExit: raise except: + if softspace(sys.stdout, 0): + print + try: + sys.stdout.flush() + except: + pass self.showtraceback() else: if softspace(sys.stdout, 0): diff --git a/lib-python/2.7/fractions.py b/lib-python/2.7/fractions.py --- a/lib-python/2.7/fractions.py +++ b/lib-python/2.7/fractions.py @@ -517,8 +517,13 @@ # Get integers right. return hash(self._numerator) # Expensive check, but definitely correct. - if self == float(self): - return hash(float(self)) + # PyPy: the following 4 lines used to be almost twice slower: + # if self == float(self): + # return hash(float(self)) + f = float(self) + x, y = f.as_integer_ratio() # signs are correct: y is positive + if self._numerator == x and self._denominator == y: + return hash(f) else: # Use tuple's hash to avoid a high collision rate on # simple fractions. diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -136,9 +136,14 @@ __get_hash = __get_openssl_constructor algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) -except ImportError: +except ImportError as e: new = __py_new __get_hash = __get_builtin_constructor + # added by PyPy + import warnings + warnings.warn("The _hashlib module is not available, falling back " + "to a much slower implementation (%s)" % str(e), + RuntimeWarning) for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py --- a/lib-python/2.7/opcode.py +++ b/lib-python/2.7/opcode.py @@ -194,5 +194,6 @@ def_op('CALL_METHOD', 202) # #args not including 'self' def_op('BUILD_LIST_FROM_ARG', 203) jrel_op('JUMP_IF_NOT_DEBUG', 204) # jump over assert statements +def_op('LOAD_REVDB_VAR', 205) # reverse debugger (syntax example: $5) del def_op, name_op, jrel_op, jabs_op diff --git a/lib-python/2.7/re.py b/lib-python/2.7/re.py --- a/lib-python/2.7/re.py +++ b/lib-python/2.7/re.py @@ -225,7 +225,7 @@ _pattern_type = type(sre_compile.compile("", 0)) -_MAXCACHE = 100 +_MAXCACHE = 1000 def _compile(*key): # internal: compile pattern diff --git a/lib-python/2.7/shutil.py b/lib-python/2.7/shutil.py --- a/lib-python/2.7/shutil.py +++ b/lib-python/2.7/shutil.py @@ -396,17 +396,21 @@ return archive_name -def _call_external_zip(base_dir, zip_filename, verbose=False, dry_run=False): +def _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger): # XXX see if we want to keep an external call here if verbose: zipoptions = "-r" else: zipoptions = "-rq" - from distutils.errors import DistutilsExecError - from distutils.spawn import spawn + cmd = ["zip", zipoptions, zip_filename, base_dir] + if logger is not None: + logger.info(' '.join(cmd)) + if dry_run: + return + import subprocess try: - spawn(["zip", zipoptions, zip_filename, base_dir], dry_run=dry_run) - except DistutilsExecError: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: # XXX really should distinguish between "couldn't find # external 'zip' command" and "zip failed". raise ExecError, \ @@ -440,7 +444,7 @@ zipfile = None if zipfile is None: - _call_external_zip(base_dir, zip_filename, verbose, dry_run) + _call_external_zip(base_dir, zip_filename, verbose, dry_run, logger) else: if logger is not None: logger.info("creating '%s' and adding '%s' to it", diff --git a/lib-python/2.7/test/test_eof.py b/lib-python/2.7/test/test_eof.py --- a/lib-python/2.7/test/test_eof.py +++ b/lib-python/2.7/test/test_eof.py @@ -5,7 +5,7 @@ class EOFTestCase(unittest.TestCase): def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" + expect = "end of line (EOL) while scanning string literal (, line 1)" try: eval("""'this is a test\ """) @@ -15,7 +15,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " + expect = ("end of file (EOF) while scanning triple-quoted string literal " "(, line 1)") try: eval("""'''this is a test""") diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py --- a/lib-python/2.7/test/test_generators.py +++ b/lib-python/2.7/test/test_generators.py @@ -398,7 +398,10 @@ 0 >>> type(i.gi_frame) ->>> i.gi_running = 42 + +PyPy prints "readonly attribute 'gi_running'" so ignore the exception detail + +>>> i.gi_running = 42 # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... TypeError: readonly attribute diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py --- a/lib-python/2.7/test/test_genexps.py +++ b/lib-python/2.7/test/test_genexps.py @@ -87,7 +87,7 @@ >>> dict(a = i for i in xrange(10)) Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax (expected ')') Verify that parenthesis are required when used as a keyword argument value diff --git a/lib-python/2.7/test/test_inspect.py b/lib-python/2.7/test/test_inspect.py --- a/lib-python/2.7/test/test_inspect.py +++ b/lib-python/2.7/test/test_inspect.py @@ -45,6 +45,9 @@ git = mod.StupidGit() +class ExampleClassWithSlot(object): + __slots__ = 'myslot' + class IsTestBase(unittest.TestCase): predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, inspect.isframe, inspect.isfunction, inspect.ismethod, @@ -96,7 +99,11 @@ else: self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals)) if hasattr(types, 'MemberDescriptorType'): - self.istest(inspect.ismemberdescriptor, 'type(lambda: None).func_globals') + # App-level slots are member descriptors on both PyPy and + # CPython, but the various built-in attributes are all + # getsetdescriptors on PyPy. So check ismemberdescriptor() + # with an app-level slot. + self.istest(inspect.ismemberdescriptor, 'ExampleClassWithSlot.myslot') else: self.assertFalse(inspect.ismemberdescriptor(type(lambda: None).func_globals)) diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py --- a/lib-python/2.7/test/test_traceback.py +++ b/lib-python/2.7/test/test_traceback.py @@ -123,10 +123,7 @@ self.assertEqual(len(err), 4) self.assertEqual(err[1].strip(), "print(2)") self.assertIn("^", err[2]) - if check_impl_detail(): - self.assertEqual(err[1].find("p"), err[2].find("^")) - if check_impl_detail(pypy=True): - self.assertEqual(err[1].find("2)") + 1, err[2].find("^")) + self.assertEqual(err[1].find("p"), err[2].find("^")) def test_base_exception(self): # Test that exceptions derived from BaseException are formatted right diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -351,6 +351,21 @@ # forward-compatibility reasons we do the same. waiter.acquire() gotit = True + except AttributeError: + # someone patched the 'waiter' class, probably. + # Fall back to the standard CPython logic. + # See the CPython lib for the comments about it... + endtime = _time() + timeout + delay = 0.0005 # 500 us -> initial delay of 1 ms + while True: + gotit = waiter.acquire(0) + if gotit: + break + remaining = endtime - _time() + if remaining <= 0: + break + delay = min(delay * 2, remaining, .05) + _sleep(delay) else: gotit = waiter.acquire(False) if not gotit: diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py --- a/lib-python/2.7/types.py +++ b/lib-python/2.7/types.py @@ -83,9 +83,19 @@ DictProxyType = type(TypeType.__dict__) NotImplementedType = type(NotImplemented) -# For Jython, the following two types are identical +# +# On CPython, FunctionType.__code__ is a 'getset_descriptor', but +# FunctionType.__globals__ is a 'member_descriptor', just like app-level +# slots. On PyPy, all descriptors of built-in types are +# 'getset_descriptor', but the app-level slots are 'member_descriptor' +# as well. (On Jython the situation might still be different.) +# +# Note that MemberDescriptorType was equal to GetSetDescriptorType in +# PyPy <= 6.0. +# GetSetDescriptorType = type(FunctionType.func_code) -MemberDescriptorType = type(FunctionType.func_globals) +class _C(object): __slots__ = 's' +MemberDescriptorType = type(_C.s) del sys, _f, _g, _C, _x # Not for export diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -82,8 +82,11 @@ def _CData_output(self, resarray, base=None, index=-1): from _rawffi.alt import types # If a char_p or unichar_p is received, skip the string interpretation - if base._ffiargtype != types.Pointer(types.char_p) and \ - base._ffiargtype != types.Pointer(types.unichar_p): + try: + deref = type(base)._deref_ffiargtype() + except AttributeError: + deref = None + if deref != types.char_p and deref != types.unichar_p: # this seems to be a string if we're array of char, surprise! from ctypes import c_char, c_wchar if self._type_ is c_char: @@ -120,6 +123,12 @@ value = self(*value) return _CDataMeta.from_param(self, value) + def _build_ffiargtype(self): + return _ffi.types.Pointer(self._type_.get_ffi_argtype()) + + def _deref_ffiargtype(self): + return self._type_.get_ffi_argtype() + def array_get_slice_params(self, index): if hasattr(self, '_length_'): start, stop, step = index.indices(self._length_) @@ -174,6 +183,7 @@ self._buffer = self._ffiarray(self._length_, autofree=True) for i, arg in enumerate(args): self[i] = arg + _init_no_arg_ = __init__ def _fix_index(self, index): if index < 0: @@ -248,6 +258,5 @@ _type_ = base ) cls = ArrayMeta(name, (Array,), tpdict) - cls._ffiargtype = _ffi.types.Pointer(base.get_ffi_argtype()) ARRAY_CACHE[key] = cls return cls diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py --- a/lib_pypy/_ctypes/basics.py +++ b/lib_pypy/_ctypes/basics.py @@ -49,10 +49,13 @@ else: return self.from_param(as_parameter) + def _build_ffiargtype(self): + return _shape_to_ffi_type(self._ffiargshape_) + def get_ffi_argtype(self): if self._ffiargtype: return self._ffiargtype - self._ffiargtype = _shape_to_ffi_type(self._ffiargshape_) + self._ffiargtype = self._build_ffiargtype() return self._ffiargtype def _CData_output(self, resbuffer, base=None, index=-1): @@ -117,7 +120,7 @@ raise ValueError( "Buffer size too small (%d instead of at least %d bytes)" % (len(buf) + offset, size + offset)) - result = self() + result = self._newowninstance_() dest = result._buffer.buffer try: raw_addr = buf._pypy_raw_address() @@ -128,6 +131,11 @@ memmove(dest, raw_addr, size) return result + def _newowninstance_(self): + result = self.__new__(self) + result._init_no_arg_() + return result + class CArgObject(object): """ simple wrapper around buffer, just for the case of freeing @@ -159,6 +167,7 @@ def __init__(self, *args, **kwds): raise TypeError("%s has no type" % (type(self),)) + _init_no_arg_ = __init__ def _ensure_objects(self): if '_objects' not in self.__dict__: diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -268,6 +268,7 @@ return raise TypeError("Unknown constructor %s" % (args,)) + _init_no_arg_ = __init__ def _wrap_callable(self, to_call, argtypes): def f(*args): @@ -485,6 +486,8 @@ return cobj, cobj._to_ffi_param(), type(cobj) def _convert_args_for_callback(self, argtypes, args): + from _ctypes.structure import StructOrUnion + # assert len(argtypes) == len(args) newargs = [] for argtype, arg in zip(argtypes, args): @@ -494,6 +497,10 @@ param = param._get_buffer_value() elif self._is_primitive(argtype): param = param.value + elif isinstance(param, StructOrUnion): # not a *pointer* to struct + newparam = StructOrUnion.__new__(type(param)) + param._copy_to(newparam._buffer.buffer) + param = newparam newargs.append(param) return newargs @@ -557,7 +564,7 @@ keepalive, newarg, newargtype = self._conv_param(argtype, defval) else: import ctypes - val = argtype._type_() + val = argtype._type_._newowninstance_() keepalive = None newarg = ctypes.byref(val) newargtype = type(newarg) diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py --- a/lib_pypy/_ctypes/pointer.py +++ b/lib_pypy/_ctypes/pointer.py @@ -67,10 +67,18 @@ self._buffer = ffiarray(1, autofree=True) if value is not None: self.contents = value + def _init_no_arg_(self): + self._buffer = ffiarray(1, autofree=True) self._ffiarray = ffiarray self.__init__ = __init__ + self._init_no_arg_ = _init_no_arg_ self._type_ = TP - self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype()) + + def _build_ffiargtype(self): + return _ffi.types.Pointer(self._type_.get_ffi_argtype()) + + def _deref_ffiargtype(self): + return self._type_.get_ffi_argtype() from_address = cdata_from_address @@ -132,27 +140,21 @@ if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): raise TypeError("cast() argument 2 must be a pointer type, not %s" % (tp,)) + result = tp._newowninstance_() if isinstance(obj, (int, long)): - result = tp() result._buffer[0] = obj return result elif obj is None: - result = tp() return result elif isinstance(obj, Array): - ptr = tp.__new__(tp) - ptr._buffer = tp._ffiarray(1, autofree=True) - ptr._buffer[0] = obj._buffer - result = ptr + result._buffer[0] = obj._buffer elif isinstance(obj, bytes): - result = tp() result._buffer[0] = buffer(obj)._pypy_raw_address() return result elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) else: - result = tp() result._buffer[0] = obj._buffer[0] # The casted objects '_objects' member: diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py --- a/lib_pypy/_ctypes/primitive.py +++ b/lib_pypy/_ctypes/primitive.py @@ -390,11 +390,14 @@ self._buffer = self._ffiarray(1, autofree=True) if value is not DEFAULT_VALUE: self.value = value + _init_no_arg_ = __init__ def _ensure_objects(self): - if self._type_ not in 'zZP': - assert self._objects is None - return self._objects + # No '_objects' is the common case for primitives. Examples + # where there is an _objects is if _type in 'zZP', or if + # self comes from 'from_buffer(buf)'. See module/test_lib_pypy/ + # ctypes_test/test_buffers.py: test_from_buffer_keepalive. + return getattr(self, '_objects', None) def _getvalue(self): return self._buffer[0] diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -160,6 +160,10 @@ raise AttributeError("_fields_ is final") if self in [f[1] for f in value]: raise AttributeError("Structure or union cannot contain itself") + if self._ffiargtype is not None: + raise NotImplementedError("Too late to set _fields_: we already " + "said to libffi that the structure type %s is opaque" + % (self,)) names_and_fields( self, value, self.__bases__[0], @@ -277,6 +281,7 @@ self.__setattr__(name, arg) for name, arg in kwds.items(): self.__setattr__(name, arg) + _init_no_arg_ = __init__ def _subarray(self, fieldtype, name): """Return a _rawffi array of length 1 whose address is the same as diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -2,7 +2,7 @@ import tempfile, binascii -def get_hashed_dir(cfile): +def _get_hashed_filename(cfile): with open(cfile,'r') as fid: content = fid.read() # from cffi's Verifier() @@ -21,10 +21,28 @@ username = os.environ['USERNAME'] #windows except KeyError: username = os.getuid() - output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % ( + return tempfile.gettempdir() + os.path.sep + 'testcapi_%s_%s%s' % ( username, k1, k2) - if not os.path.exists(output_dir): + +def get_hashed_dir(cfile): + hashed_fn = _get_hashed_filename(cfile) + try: + with open(hashed_fn) as f: + dirname = f.read(1024) + except IOError: + dirname = '' + tmpdir = tempfile.gettempdir() + if (not dirname or '/' in dirname or '\\' in dirname or '\x00' in dirname + or not os.path.isdir(os.path.join(tmpdir, dirname))): + dirname = binascii.hexlify(os.urandom(8)) + if not isinstance(dirname, str): # Python 3 + dirname = dirname.decode('ascii') + dirname = 'testcapi_' + dirname + output_dir = os.path.join(tmpdir, dirname) + try: os.mkdir(output_dir) + except OSError: + pass return output_dir @@ -34,13 +52,12 @@ return ext -def compile_shared(csource, modulename, output_dir=None): +def compile_shared(csource, modulename, output_dir): """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module, and import it. """ thisdir = os.path.dirname(__file__) - if output_dir is None: - output_dir = tempfile.mkdtemp() + assert output_dir is not None from distutils.ccompiler import new_compiler @@ -85,4 +102,16 @@ # Now import the newly created library, it will replace the original # module in sys.modules fp, filename, description = imp.find_module(modulename, path=[output_dir]) - imp.load_module(modulename, fp, filename, description) + with fp: + imp.load_module(modulename, fp, filename, description) + + # If everything went fine up to now, write the name of this new + # directory to 'hashed_fn', for future processes (and to avoid a + # growing number of temporary directories that are not completely + # obvious to clean up on Windows) + hashed_fn = _get_hashed_filename(os.path.join(thisdir, csource)) + try: + with open(hashed_fn, 'w') as f: + f.write(os.path.basename(output_dir)) + except IOError: + pass diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -153,9 +153,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) + res = factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements) add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements) + return res def _unicode_text_factory(x): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.4 +Version: 1.12.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.4" -__version_info__ = (1, 11, 4) +__version__ = "1.12.0" +__version_info__ = (1, 12, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h --- a/lib_pypy/cffi/_cffi_errors.h +++ b/lib_pypy/cffi/_cffi_errors.h @@ -50,7 +50,9 @@ "import sys\n" "class FileLike:\n" " def write(self, x):\n" - " of.write(x)\n" + " try:\n" + " of.write(x)\n" + " except: pass\n" " self.buf += x\n" "fl = FileLike()\n" "fl.buf = ''\n" diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -8,20 +8,43 @@ the same works for the other two macros. Py_DEBUG implies them, but not the other way around. - Issue #350 is still open: on Windows, the code here causes it to link - with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was - attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv - does not make PYTHON3.DLL available, and so the "correctly" compiled - version would not run inside a virtualenv. We will re-apply the fix - after virtualenv has been fixed for some time. For explanation, see - issue #355. For a workaround if you want PYTHON3.DLL and don't worry - about virtualenv, see issue #350. See also 'py_limited_api' in - setuptools_ext.py. + The implementation is messy (issue #350): on Windows, with _MSC_VER, + we have to define Py_LIMITED_API even before including pyconfig.h. + In that case, we guess what pyconfig.h will do to the macros above, + and check our guess after the #include. + + Note that on Windows, with CPython 3.x, you need virtualenv version + >= 16.0.0. Older versions don't copy PYTHON3.DLL. As a workaround + you can remove the definition of Py_LIMITED_API here. + + See also 'py_limited_api' in cffi/setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) -# include -# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) -# define Py_LIMITED_API +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) +# define Py_LIMITED_API +# endif # endif #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -146,32 +146,6 @@ PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; -#if PY_MAJOR_VERSION >= 3 - /* see comments in _cffi_carefully_make_gil() about the - Python2/Python3 difference - */ -#else - /* Acquire the GIL. We have no threadstate here. If Python is - already initialized, it is possible that there is already one - existing for this thread, but it is not made current now. - */ - PyEval_AcquireLock(); - - _cffi_py_initialize(); - - /* The Py_InitializeEx() sometimes made a threadstate for us, but - not always. Indeed Py_InitializeEx() could be called and do - nothing. So do we have a threadstate, or not? We don't know, - but we can replace it with NULL in all cases. - */ - (void)PyThreadState_Swap(NULL); - - /* Now we can release the GIL and re-acquire immediately using the - logic of PyGILState(), which handles making or installing the - correct threadstate. - */ - PyEval_ReleaseLock(); -#endif state = PyGILState_Ensure(); /* Call the initxxx() function from the present module. It will @@ -247,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.4" + "\ncompiled with cffi version: 1.12.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -278,16 +252,14 @@ that we don't hold the GIL before (if it exists), and we don't hold it afterwards. - What it really does is completely different in Python 2 and - Python 3. + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) - Python 2 - ======== - - Initialize the GIL, without initializing the rest of Python, - by calling PyEval_InitThreads(). - - PyEval_InitThreads() must not be called concurrently at all. + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from @@ -297,18 +269,6 @@ string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) - - Python 3 - ======== - - In Python 3, PyEval_InitThreads() cannot be called before - Py_InitializeEx() any more. So this function calls - Py_InitializeEx() first. It uses the same obscure logic to - make sure we never call it concurrently. - - Arguably, this is less good on the spinlock, because - Py_InitializeEx() takes much longer to run than - PyEval_InitThreads(). But I didn't find a way around it. */ #ifdef WITH_THREAD @@ -332,8 +292,7 @@ } #endif -#if PY_MAJOR_VERSION >= 3 - /* Python 3: call Py_InitializeEx() */ + /* call Py_InitializeEx() */ { PyGILState_STATE state = PyGILState_UNLOCKED; if (!Py_IsInitialized()) @@ -344,17 +303,6 @@ PyEval_InitThreads(); PyGILState_Release(state); } -#else - /* Python 2: call PyEval_InitThreads() */ -# ifdef WITH_THREAD - if (!PyEval_ThreadsInitialized()) { - PyEval_InitThreads(); /* makes the GIL */ - PyEval_ReleaseLock(); /* then release it */ - } - /* else: there is already a GIL, but we still needed to do the - spinlock dance to make sure that we see it as fully ready */ -# endif -#endif #ifdef WITH_THREAD /* release the lock */ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -96,18 +96,21 @@ self.CData, self.CType = backend._get_types() self.buffer = backend.buffer - def cdef(self, csource, override=False, packed=False): + def cdef(self, csource, override=False, packed=False, pack=None): """Parse the given C source. This registers all declared functions, types, and global variables. The functions and global variables can then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'. The types can be used in 'ffi.new()' and other functions. If 'packed' is specified as True, all structs declared inside this cdef are packed, i.e. laid out without any field alignment at all. + Alternatively, 'pack' can be a small integer, and requests for + alignment greater than that are ignored (pack=1 is equivalent to + packed=True). """ - self._cdef(csource, override=override, packed=packed) + self._cdef(csource, override=override, packed=packed, pack=pack) - def embedding_api(self, csource, packed=False): - self._cdef(csource, packed=packed, dllexport=True) + def embedding_api(self, csource, packed=False, pack=None): + self._cdef(csource, packed=packed, pack=pack, dllexport=True) if self._embedding is None: self._embedding = '' @@ -143,6 +146,13 @@ self._libraries.append(lib) return lib + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + def _typeof_locked(self, cdecl): # call me with the lock! key = cdecl @@ -898,6 +908,9 @@ return addressof_var(name) raise AttributeError("cffi library has no function or " "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() # if libname is not None: try: diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -636,6 +636,10 @@ if isinstance(init, bytes): init = [init[i:i+1] for i in range(len(init))] else: + if isinstance(init, CTypesGenericArray): + if (len(init) != len(blob) or + not isinstance(init, CTypesArray)): + raise TypeError("length/type mismatch: %s" % (init,)) init = tuple(init) if len(init) > len(blob): raise IndexError("too many initializers") @@ -730,7 +734,8 @@ return self._new_struct_or_union('union', name, ctypes.Union) def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp, - totalsize=-1, totalalignment=-1, sflags=0): + totalsize=-1, totalalignment=-1, sflags=0, + pack=0): if totalsize >= 0 or totalalignment >= 0: raise NotImplementedError("the ctypes backend of CFFI does not support " "structures completed by verify(); please " @@ -751,6 +756,8 @@ bfield_types[fname] = Ellipsis if sflags & 8: struct_or_union._pack_ = 1 + elif pack: + struct_or_union._pack_ = pack struct_or_union._fields_ = cfields CTypesStructOrUnion._bfield_types = bfield_types # diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -306,11 +306,25 @@ msg = 'parse error\n%s' % (msg,) raise CDefError(msg) - def parse(self, csource, override=False, packed=False, dllexport=False): + def parse(self, csource, override=False, packed=False, pack=None, + dllexport=False): + if packed: + if packed != True: + raise ValueError("'packed' should be False or True; use " + "'pack' to give another value") + if pack: + raise ValueError("cannot give both 'pack' and 'packed'") + pack = 1 + elif pack: + if pack & (pack - 1): + raise ValueError("'pack' must be a power of two, not %r" % + (pack,)) + else: + pack = 0 prev_options = self._options try: self._options = {'override': override, - 'packed': packed, + 'packed': pack, 'dllexport': dllexport} self._internal_parse(csource) finally: diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -342,7 +342,7 @@ fixedlayout = None completed = 0 partial = False - packed = False + packed = 0 def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None): self.name = name @@ -352,21 +352,20 @@ self.fldquals = fldquals self.build_c_name_with_marker() - def has_anonymous_struct_fields(self): - if self.fldtypes is None: - return False - for name, type in zip(self.fldnames, self.fldtypes): - if name == '' and isinstance(type, StructOrUnion): - return True - return False + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type - def enumfields(self): + def enumfields(self, expand_anonymous_struct_union=True): fldquals = self.fldquals if fldquals is None: fldquals = (0,) * len(self.fldnames) for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, self.fldbitsize, fldquals): - if name == '' and isinstance(type, StructOrUnion): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): # nested anonymous struct/union for result in type.enumfields(): yield result @@ -415,11 +414,14 @@ fldtypes = [tp.get_cached_btype(ffi, finishlist) for tp in self.fldtypes] lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) - sflags = 0 + extra_flags = () if self.packed: - sflags = 8 # SF_PACKED + if self.packed == 1: + extra_flags = (8,) # SF_PACKED + else: + extra_flags = (0, self.packed) ffi._backend.complete_struct_or_union(BType, lst, self, - -1, -1, sflags) + -1, -1, *extra_flags) # else: fldtypes = [] diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -836,6 +836,10 @@ def _struct_collecttype(self, tp): self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) def _struct_decl(self, tp, cname, approxname): if tp.fldtypes is None: @@ -884,11 +888,17 @@ named_ptr not in self.ffi._parser._included_declarations)): if tp.fldtypes is None: pass # opaque - elif tp.partial or tp.has_anonymous_struct_fields(): + elif tp.partial or any(tp.anonymous_struct_fields()): pass # field layout obtained silently from the C compiler else: flags.append("_CFFI_F_CHECK_FIELDS") if tp.packed: + if tp.packed > 1: + raise NotImplementedError( + "%r is declared with 'pack=%r'; only 0 or 1 are " + "supported in API mode (try to use \"...;\", which " + "does not require a 'pack' declaration)" % + (tp, tp.packed)) flags.append("_CFFI_F_PACKED") else: flags.append("_CFFI_F_EXTERNAL") @@ -896,7 +906,8 @@ flags = '|'.join(flags) or '0' c_fields = [] if reason_for_not_expanding is None: - enumfields = list(tp.enumfields()) + expand_anonymous_struct_union = not self.target_is_python + enumfields = list(tp.enumfields(expand_anonymous_struct_union)) for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) self._check_not_opaque(fldtype, diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -143,8 +143,8 @@ def _add_py_module(dist, ffi, module_name): from distutils.dir_util import mkpath - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext from distutils import log from cffi import recompiler @@ -162,8 +162,31 @@ module_path = module_name.split('.') module_path[-1] += '.py' generate_mod(os.path.join(self.build_lib, *module_path)) + def get_source_files(self): + # This is called from 'setup.py sdist' only. Exclude + # the generate .py module in this case. + saved_py_modules = self.py_modules + try: + if saved_py_modules: + self.py_modules = [m for m in saved_py_modules + if m != module_name] + return base_class.get_source_files(self) + finally: + self.py_modules = saved_py_modules dist.cmdclass['build_py'] = build_py_make_mod + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + # Then we need to hack more in get_source_files(); see above. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + # the following is only for "build_ext -i" base_class_2 = dist.cmdclass.get('build_ext', build_ext) class build_ext_make_mod(base_class_2): diff --git a/lib_pypy/dbm.py b/lib_pypy/dbm.py --- a/lib_pypy/dbm.py +++ b/lib_pypy/dbm.py @@ -157,7 +157,14 @@ def open(filename, flag='r', mode=0666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") openflag = 0 diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py --- a/lib_pypy/grp.py +++ b/lib_pypy/grp.py @@ -4,6 +4,8 @@ from _pwdgrp_cffi import ffi, lib import _structseq +import thread +_lock = thread.allocate_lock() try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f @@ -33,32 +35,35 @@ @builtinify def getgrgid(gid): - res = lib.getgrgid(gid) - if not res: - # XXX maybe check error eventually - raise KeyError(gid) - return _group_from_gstruct(res) + with _lock: + res = lib.getgrgid(gid) + if not res: + # XXX maybe check error eventually + raise KeyError(gid) + return _group_from_gstruct(res) @builtinify def getgrnam(name): if not isinstance(name, basestring): raise TypeError("expected string") name = str(name) - res = lib.getgrnam(name) - if not res: - raise KeyError("'getgrnam(): name not found: %s'" % name) - return _group_from_gstruct(res) + with _lock: + res = lib.getgrnam(name) + if not res: + raise KeyError("'getgrnam(): name not found: %s'" % name) + return _group_from_gstruct(res) @builtinify def getgrall(): - lib.setgrent() lst = [] - while 1: - p = lib.getgrent() - if not p: - break - lst.append(_group_from_gstruct(p)) - lib.endgrent() + with _lock: + lib.setgrent() + while 1: + p = lib.getgrent() + if not p: + break + lst.append(_group_from_gstruct(p)) + lib.endgrent() return lst __all__ = ('struct_group', 'getgrgid', 'getgrnam', 'getgrall') diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py --- a/lib_pypy/pwd.py +++ b/lib_pypy/pwd.py @@ -12,6 +12,8 @@ from _pwdgrp_cffi import ffi, lib import _structseq +import thread +_lock = thread.allocate_lock() try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f @@ -55,10 +57,11 @@ Return the password database entry for the given numeric user ID. See pwd.__doc__ for more on password database entries. """ - pw = lib.getpwuid(uid) - if not pw: - raise KeyError("getpwuid(): uid not found: %s" % uid) - return _mkpwent(pw) + with _lock: + pw = lib.getpwuid(uid) + if not pw: + raise KeyError("getpwuid(): uid not found: %s" % uid) + return _mkpwent(pw) @builtinify def getpwnam(name): @@ -71,10 +74,11 @@ if not isinstance(name, basestring): raise TypeError("expected string") name = str(name) - pw = lib.getpwnam(name) - if not pw: - raise KeyError("getpwname(): name not found: %s" % name) - return _mkpwent(pw) + with _lock: + pw = lib.getpwnam(name) + if not pw: + raise KeyError("getpwname(): name not found: %s" % name) + return _mkpwent(pw) @builtinify def getpwall(): @@ -84,13 +88,14 @@ See pwd.__doc__ for more on password database entries. """ users = [] - lib.setpwent() - while True: - pw = lib.getpwent() - if not pw: - break - users.append(_mkpwent(pw)) - lib.endpwent() + with _lock: + lib.setpwent() + while True: + pw = lib.getpwent() + if not pw: + break + users.append(_mkpwent(pw)) + lib.endpwent() return users __all__ = ('struct_passwd', 'getpwuid', 'getpwnam', 'getpwall') diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -70,8 +70,8 @@ pass def register(self, fd, flag): self.fd = fd - def poll(self, timeout=None): - r,w,e = select.select([self.fd],[],[],timeout) + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r,w,e = select.select([self.fd],[],[]) return r POLLIN = getattr(select, "POLLIN", None) diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -39,14 +39,10 @@ "_csv", "_cppyy", "_pypyjson", "_jitlog" ]) -from rpython.jit.backend import detect_cpu -try: - if detect_cpu.autodetect().startswith('x86'): - if not sys.platform.startswith('openbsd'): - working_modules.add('_vmprof') - working_modules.add('faulthandler') -except detect_cpu.ProcessorAutodetectError: - pass +import rpython.rlib.rvmprof.cintf +if rpython.rlib.rvmprof.cintf.IS_SUPPORTED: + working_modules.add('_vmprof') + working_modules.add('faulthandler') translation_modules = default_modules.copy() translation_modules.update([ @@ -57,6 +53,11 @@ "termios", "_minimal_curses", ]) +reverse_debugger_disable_modules = set([ + "_continuation", "_vmprof", "_multiprocessing", + "micronumpy", + ]) + # XXX this should move somewhere else, maybe to platform ("is this posixish" # check or something) if sys.platform == "win32": @@ -292,6 +293,9 @@ modules = working_modules.copy() if config.translation.sandbox: modules = default_modules + if config.translation.reverse_debugger: + for mod in reverse_debugger_disable_modules: + setattr(config.objspace.usemodules, mod, False) # ignore names from 'essential_modules', notably 'exceptions', which # may not be present in config.objspace.usemodules at all modules = [name for name in modules if name not in essential_modules] @@ -310,3 +314,4 @@ parser = to_optparse(config) #, useoptions=["translation.*"]) option, args = parser.parse_args() print config + print working_modules diff --git a/pypy/doc/architecture.rst b/pypy/doc/architecture.rst --- a/pypy/doc/architecture.rst +++ b/pypy/doc/architecture.rst @@ -73,3 +73,63 @@ This division between bytecode evaluator and object space gives a lot of flexibility. One can plug in different :doc:`object spaces ` to get different or enriched behaviours of the Python objects. + +Layers +------ + +RPython +~~~~~~~ +:ref:`RPython ` is the language in which we write interpreters. +Not the entire PyPy project is written in RPython, only the parts that are +compiled in the translation process. The interesting point is that RPython +has no parser, it's compiled from the live python objects, which makes it +possible to do all kinds of metaprogramming during import time. In short, +Python is a meta programming language for RPython. + +The RPython standard library is to be found in the ``rlib`` subdirectory. + +Consult `Getting Started with RPython`_ for further reading + +Translation +~~~~~~~~~~~ +The translation toolchain - this is the part that takes care of translating +RPython to flow graphs and then to C. There is more in the +:doc:`architecture ` document written about it. + +It lives in the ``rpython`` directory: ``flowspace``, ``annotator`` +and ``rtyper``. + +PyPy Interpreter +~~~~~~~~~~~~~~~~ +This is in the ``pypy`` directory. ``pypy/interpreter`` is a standard +interpreter for Python written in RPython. The fact that it is +RPython is not apparent at first. Built-in modules are written in +``pypy/module/*``. Some modules that CPython implements in C are +simply written in pure Python; they are in the top-level ``lib_pypy`` +directory. The standard library of Python (with a few changes to +accomodate PyPy) is in ``lib-python``. + +JIT Compiler +~~~~~~~~~~~~ +:ref:`Just-in-Time Compiler (JIT) `: we have a tracing JIT that traces the +interpreter written in RPython, rather than the user program that it +interprets. As a result it applies to any interpreter, i.e. any +language. But getting it to work correctly is not trivial: it +requires a small number of precise "hints" and possibly some small +refactorings of the interpreter. The JIT itself also has several +almost-independent parts: the tracer itself in ``rpython/jit/metainterp``, the +optimizer in ``rpython/jit/metainterp/optimizer`` that optimizes a list of +residual operations, and the backend in ``rpython/jit/backend/`` +that turns it into machine code. Writing a new backend is a +traditional way to get into the project. + +Garbage Collectors +~~~~~~~~~~~~~~~~~~ +Garbage Collectors (GC): as you may notice if you are used to CPython's +C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code. +:ref:`rpython:garbage-collection` is inserted +during translation. Moreover, this is not reference counting; it is a real +GC written as more RPython code. The best one we have so far is in +``rpython/memory/gc/incminimark.py``. + +.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst --- a/pypy/doc/build.rst +++ b/pypy/doc/build.rst @@ -267,14 +267,14 @@ * PyPy 2.5.1 or earlier: normal users would see permission errors. Installers need to run ``pypy -c "import gdbm"`` and other similar commands at install time; the exact list is in - :source:`pypy/tool/release/package.py `. Users + :source:`pypy/tool/release/package.py`. Users seeing a broken installation of PyPy can fix it after-the-fact if they have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``. * PyPy 2.6 and later: anyone would get ``ImportError: no module named _gdbm_cffi``. Installers need to run ``pypy _gdbm_build.py`` in the ``lib_pypy`` directory during the installation process (plus others; - see the exact list in :source:`pypy/tool/release/package.py `). + see the exact list in :source:`pypy/tool/release/package.py`). Users seeing a broken installation of PyPy can fix it after-the-fact, by running ``pypy /path/to/lib_pypy/_gdbm_build.py``. This command produces a file diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -539,7 +539,7 @@ hg help branch -.. _official wiki: http://mercurial.selenic.com/wiki/Branch +.. _official wiki: https://www.mercurial-scm.org/wiki/ .. _using-development-tracker: @@ -547,15 +547,7 @@ Using the development bug/feature tracker ----------------------------------------- -We have a `development tracker`_, based on Richard Jones' -`roundup`_ application. You can file bugs, -feature requests or see what's going on -for the next milestone, both from an E-Mail and from a -web interface. - -.. _development tracker: https://bugs.pypy.org/ -.. _roundup: http://roundup.sourceforge.net/ - +We use bitbucket for :source:`issues` tracking and :source:`pull-requests`. .. _testing: diff --git a/pypy/doc/commandline_ref.rst b/pypy/doc/commandline_ref.rst --- a/pypy/doc/commandline_ref.rst +++ b/pypy/doc/commandline_ref.rst @@ -8,3 +8,4 @@ :maxdepth: 1 man/pypy.1.rst + man/pypy3.1.rst diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -66,9 +66,9 @@ # built documents. # # The short X.Y version. -version = '5.8' +version = '6.0' # The full version, including alpha/beta/rc tags. -release = '5.8.0' +release = '6.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/config/objspace.disable_entrypoints.txt b/pypy/doc/config/objspace.disable_entrypoints.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.fstrings.txt b/pypy/doc/config/objspace.fstrings.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.hash.txt b/pypy/doc/config/objspace.hash.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.usemodules._cppyy.txt b/pypy/doc/config/objspace.usemodules._cppyy.txt new file mode 100644 --- /dev/null +++ b/pypy/doc/config/objspace.usemodules._cppyy.txt @@ -0,0 +1,1 @@ +The internal backend for cppyy diff --git a/pypy/doc/config/objspace.usemodules._frozen_importlib.txt b/pypy/doc/config/objspace.usemodules._frozen_importlib.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.usemodules._jitlog.txt b/pypy/doc/config/objspace.usemodules._jitlog.txt new file mode 100644 diff --git a/pypy/doc/config/objspace.usemodules._rawffi.txt b/pypy/doc/config/objspace.usemodules._rawffi.txt --- a/pypy/doc/config/objspace.usemodules._rawffi.txt +++ b/pypy/doc/config/objspace.usemodules._rawffi.txt @@ -1,3 +1,3 @@ -An experimental module providing very low-level interface to +A module providing very low-level interface to C-level libraries, for use when implementing ctypes, not -intended for a direct use at all. \ No newline at end of file +intended for a direct use at all. diff --git a/pypy/doc/config/objspace.usemodules.cpyext.txt b/pypy/doc/config/objspace.usemodules.cpyext.txt --- a/pypy/doc/config/objspace.usemodules.cpyext.txt +++ b/pypy/doc/config/objspace.usemodules.cpyext.txt @@ -1,1 +1,1 @@ -Use (experimental) cpyext module, that tries to load and run CPython extension modules +Use cpyext module to load and run CPython extension modules diff --git a/pypy/doc/config/objspace.usemodules.faulthandler.txt b/pypy/doc/config/objspace.usemodules.faulthandler.txt new file mode 100644 diff --git a/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt b/pypy/doc/config/translation.backendopt.replace_we_are_jitted.txt new file mode 100644 diff --git a/pypy/doc/config/translation.jit_opencoder_model.txt b/pypy/doc/config/translation.jit_opencoder_model.txt new file mode 100644 diff --git a/pypy/doc/config/translation.keepgoing.txt b/pypy/doc/config/translation.keepgoing.txt new file mode 100644 diff --git a/pypy/doc/config/translation.libname.txt b/pypy/doc/config/translation.libname.txt new file mode 100644 diff --git a/pypy/doc/config/translation.lto.txt b/pypy/doc/config/translation.lto.txt new file mode 100644 diff --git a/pypy/doc/config/translation.profoptargs.txt b/pypy/doc/config/translation.profoptargs.txt new file mode 100644 diff --git a/pypy/doc/config/translation.reverse_debugger.txt b/pypy/doc/config/translation.reverse_debugger.txt new file mode 100644 diff --git a/pypy/doc/config/translation.split_gc_address_space.txt b/pypy/doc/config/translation.split_gc_address_space.txt new file mode 100644 diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/contributing.rst @@ -0,0 +1,472 @@ +Contributing Guidelines +=========================== + +.. contents:: + +PyPy is a very large project that has a reputation of being hard to dive into. +Some of this fame is warranted, some of it is purely accidental. There are three +important lessons that everyone willing to contribute should learn: + +* PyPy has layers. There are many pieces of architecture that are very well + separated from each other. More about this below, but often the manifestation + of this is that things are at a different layer than you would expect them + to be. For example if you are looking for the JIT implementation, you will + not find it in the implementation of the Python programming language. + +* Because of the above, we are very serious about Test Driven Development. + It's not only what we believe in, but also that PyPy's architecture is + working very well with TDD in mind and not so well without it. Often + development means progressing in an unrelated corner, one unittest + at a time; and then flipping a giant switch, bringing it all together. + (It generally works out of the box. If it doesn't, then we didn't + write enough unit tests.) It's worth repeating - PyPy's + approach is great if you do TDD, and not so great otherwise. + +* PyPy uses an entirely different set of tools - most of them included + in the PyPy repository. There is no Makefile, nor autoconf. More below. + +The first thing to remember is that PyPy project is very different than most +projects out there. It's also different from a classic compiler project, +so academic courses about compilers often don't apply or lead in the wrong +direction. However, if you want to understand how designing & building a runtime +works in the real world then this is a great project! + +Getting involved +^^^^^^^^^^^^^^^^ + +PyPy employs a relatively standard open-source development process. You are +encouraged as a first step to join our `pypy-dev mailing list`_ and IRC channel, +details of which can be found in our :ref:`contact ` section. The folks +there are very friendly, and can point you in the right direction. + +We give out commit rights usually fairly liberally, so if you want to do something +with PyPy, you can become a committer. We also run frequent coding sprints which +are separately announced and often happen around Python conferences such as +EuroPython or PyCon. Upcoming events are usually announced on `the blog`_. + +Further Reading: :ref:`Contact ` + +.. _the blog: http://morepypy.blogspot.com +.. _pypy-dev mailing list: http://mail.python.org/mailman/listinfo/pypy-dev + + +Your first contribution +^^^^^^^^^^^^^^^^^^^^^^^ + +The first and most important rule how **not** to contribute to PyPy is +"just hacking a feature". This won't work, and you'll find your PR will typically +require a lot of re-work. There are a few reasons why not: + +* build times are large +* PyPy has very thick layer separation +* context of the cPython runtime is often required + +Instead, reach out on the dev mailing list or the IRC channel, and we're more +than happy to help! :) + +Some ideas for first contributions are: + +* Documentation - this will give you an understanding of the pypy architecture +* Test failures - find a failing test in the `nightly builds`_, and fix it +* Missing language features - these are listed in our `issue tracker`_ + +.. _nightly builds: http://buildbot.pypy.org/nightly/ +.. _issue tracker: https://bitbucket.org/pypy/pypy/issues From pypy.commits at gmail.com Thu Oct 25 09:18:16 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 25 Oct 2018 06:18:16 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: if the errorhandler changes utf8, reflect that back into w_obj Message-ID: <5bd1c298.1c69fb81.a5c91.0c53@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95241:c44ea7bcdc43 Date: 2018-10-25 16:17 +0300 http://bitbucket.org/pypy/pypy/changeset/c44ea7bcdc43/ Log: if the errorhandler changes utf8, reflect that back into w_obj diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -1224,6 +1224,7 @@ # surrogatepass? break pos = _pos + w_object = space.newtext(utf8) if errors is None or errors == 'strict': if encoding is None or encoding == 'utf-8': #if rutf8.has_surrogates(utf8): From pypy.commits at gmail.com Thu Oct 25 12:26:04 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 25 Oct 2018 09:26:04 -0700 (PDT) Subject: [pypy-commit] pypy default: Implement PyOS_InputHook in cpyext. Call it from pyrepl, once before every Message-ID: <5bd1ee9c.1c69fb81.c32bf.4df7@mx.google.com> Author: Armin Rigo Branch: Changeset: r95243:caaf91a51641 Date: 2018-10-25 18:25 +0200 http://bitbucket.org/pypy/pypy/changeset/caaf91a51641/ Log: Implement PyOS_InputHook in cpyext. Call it from pyrepl, once before every character input. It seems to me that details on when it is called are not consistent across multiple platforms of CPython; for example with the GNU readline it is called every 0.1 seconds while waiting for input. But if you compile CPython without GNU readline support, then it is only called once per character, as far as I can tell, like pyrepl does now. diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -26,6 +26,12 @@ from pyrepl.fancy_termios import tcgetattr, tcsetattr from pyrepl.console import Console, Event from pyrepl import unix_eventqueue +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -416,6 +422,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError), err: diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -109,6 +109,7 @@ '_promote' : 'interp_magic._promote', 'side_effects_ok' : 'interp_magic.side_effects_ok', 'stack_almost_full' : 'interp_magic.stack_almost_full', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -211,3 +211,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -644,6 +644,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + 'PyOS_InputHook', '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack', 'PyMem_Malloc', 'PyObject_Free', 'PyObject_GC_Del', 'PyType_GenericAlloc', '_PyObject_New', '_PyObject_NewVar', @@ -1180,6 +1181,10 @@ state.C._PyPy_object_dealloc = rffi.llexternal( '_PyPy_object_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) + FUNCPTR = lltype.Ptr(lltype.FuncType([], rffi.INT)) + state.C.get_pyos_inputhook = rffi.llexternal( + '_PyPy_get_PyOS_InputHook', [], FUNCPTR, + compilation_info=eci, _nowrapper=True) def init_function(func): @@ -1726,6 +1731,12 @@ w_mod = state.fixup_extension(name, path) return w_mod +def invoke_pyos_inputhook(space): + state = space.fromcache(State) + c_inputhook = state.C.get_pyos_inputhook() + if c_inputhook: + generic_cpy_call(space, c_inputhook) + @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h --- a/pypy/module/cpyext/include/pythonrun.h +++ b/pypy/module/cpyext/include/pythonrun.h @@ -47,6 +47,11 @@ #define Py_CompileString(str, filename, start) Py_CompileStringFlags(str, filename, start, NULL) +/* Stuff with no proper home (yet) */ +PyAPI_DATA(int) (*PyOS_InputHook)(void); +typedef int (*_pypy_pyos_inputhook)(void); +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void); + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c --- a/pypy/module/cpyext/src/missing.c +++ b/pypy/module/cpyext/src/missing.c @@ -31,3 +31,7 @@ void _Py_setfilesystemdefaultencoding(const char *enc) { Py_FileSystemDefaultEncoding = enc; } +int (*PyOS_InputHook)(void) = 0; /* only ever filled in by C extensions */ +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void) { + return PyOS_InputHook; +} diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_misc.py @@ -0,0 +1,35 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestMisc(AppTestCpythonExtensionBase): + + def test_pyos_inputhook(self): + module = self.import_extension('foo', [ + ("set_pyos_inputhook", "METH_NOARGS", + ''' + PyOS_InputHook = &my_callback; + Py_RETURN_NONE; + '''), + ("fetch_value", "METH_NOARGS", + ''' + return PyInt_FromLong(my_flag); + '''), + ], prologue=''' + static long my_flag = 0; + static int my_callback(void) { my_flag++; } + ''') + + try: + import __pypy__ + except ImportError: + skip("only runs on top of pypy") + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 0 + module.set_pyos_inputhook() # <= set + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 1 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 2 + assert module.fetch_value() == 2 From pypy.commits at gmail.com Thu Oct 25 12:33:57 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 25 Oct 2018 09:33:57 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5bd1f075.1c69fb81.60d9f.4510@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r95244:c7d3ff6df4ef Date: 2018-10-25 18:31 +0200 http://bitbucket.org/pypy/pypy/changeset/c7d3ff6df4ef/ Log: hg merge default diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -27,6 +27,12 @@ from .console import Console, Event from .unix_eventqueue import EventQueue from .trace import trace +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -76,8 +82,8 @@ pass def register(self, fd, flag): self.fd = fd - def poll(self, timeout=None): - r,w,e = select.select([self.fd],[],[],timeout) + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r,w,e = select.select([self.fd],[],[]) return r POLLIN = getattr(select, "POLLIN", None) @@ -407,6 +413,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError) as err: diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -110,6 +110,7 @@ 'stack_almost_full' : 'interp_magic.stack_almost_full', 'fsencode' : 'interp_magic.fsencode', 'fsdecode' : 'interp_magic.fsdecode', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } submodules = { diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -209,3 +209,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -647,6 +647,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + 'PyOS_InputHook', 'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree', 'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free', @@ -1183,6 +1184,10 @@ state.C._PyPy_object_dealloc = rffi.llexternal( '_PyPy_object_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) + FUNCPTR = lltype.Ptr(lltype.FuncType([], rffi.INT)) + state.C.get_pyos_inputhook = rffi.llexternal( + '_PyPy_get_PyOS_InputHook', [], FUNCPTR, + compilation_info=eci, _nowrapper=True) def init_function(func): @@ -1789,6 +1794,12 @@ return return exec_def(space, w_mod, mod_as_pyobj) +def invoke_pyos_inputhook(space): + state = space.fromcache(State) + c_inputhook = state.C.get_pyos_inputhook() + if c_inputhook: + generic_cpy_call(space, c_inputhook) + @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h --- a/pypy/module/cpyext/include/pythonrun.h +++ b/pypy/module/cpyext/include/pythonrun.h @@ -41,6 +41,11 @@ #define Py_CompileString(str, filename, start) Py_CompileStringFlags(str, filename, start, NULL) +/* Stuff with no proper home (yet) */ +PyAPI_DATA(int) (*PyOS_InputHook)(void); +typedef int (*_pypy_pyos_inputhook)(void); +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void); + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c --- a/pypy/module/cpyext/src/missing.c +++ b/pypy/module/cpyext/src/missing.c @@ -31,3 +31,7 @@ void _Py_setfilesystemdefaultencoding(const char *enc) { Py_FileSystemDefaultEncoding = enc; } +int (*PyOS_InputHook)(void) = 0; /* only ever filled in by C extensions */ +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void) { + return PyOS_InputHook; +} diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_misc.py @@ -0,0 +1,35 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestMisc(AppTestCpythonExtensionBase): + + def test_pyos_inputhook(self): + module = self.import_extension('foo', [ + ("set_pyos_inputhook", "METH_NOARGS", + ''' + PyOS_InputHook = &my_callback; + Py_RETURN_NONE; + '''), + ("fetch_value", "METH_NOARGS", + ''' + return PyLong_FromLong(my_flag); + '''), + ], prologue=''' + static long my_flag = 0; + static int my_callback(void) { my_flag++; } + ''') + + try: + import __pypy__ + except ImportError: + skip("only runs on top of pypy") + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 0 + module.set_pyos_inputhook() # <= set + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 1 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 2 + assert module.fetch_value() == 2 From pypy.commits at gmail.com Fri Oct 26 03:35:18 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 00:35:18 -0700 (PDT) Subject: [pypy-commit] pypy default: issue #2840 Message-ID: <5bd2c3b6.1c69fb81.2d050.fa04@mx.google.com> Author: Armin Rigo Branch: Changeset: r95245:a12ac2eaf1d6 Date: 2018-10-26 09:34 +0200 http://bitbucket.org/pypy/pypy/changeset/a12ac2eaf1d6/ Log: issue #2840 Don't release the GIL in the pwd module diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py --- a/pypy/module/pwd/interp_pwd.py +++ b/pypy/module/pwd/interp_pwd.py @@ -37,7 +37,8 @@ passwd_p = lltype.Ptr(config['passwd']) def external(name, args, result, **kwargs): - return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs) + return rffi.llexternal(name, args, result, compilation_info=eci, + releasegil=False, **kwargs) c_getpwuid = external("getpwuid", [uid_t], passwd_p) c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p) From pypy.commits at gmail.com Fri Oct 26 03:55:30 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 00:55:30 -0700 (PDT) Subject: [pypy-commit] cffi default: #383 Message-ID: <5bd2c872.1c69fb81.71933.f64f@mx.google.com> Author: Armin Rigo Branch: Changeset: r3170:b50894d36bae Date: 2018-10-26 09:55 +0200 http://bitbucket.org/cffi/cffi/changeset/b50894d36bae/ Log: #383 Increase the sleep time in this test diff --git a/testing/embedding/add1.py b/testing/embedding/add1.py --- a/testing/embedding/add1.py +++ b/testing/embedding/add1.py @@ -11,7 +11,7 @@ sys.stdout.write("preparing") for i in range(3): sys.stdout.flush() - time.sleep(0.02) + time.sleep(0.2) sys.stdout.write(".") sys.stdout.write("\n") diff --git a/testing/embedding/test_thread.py b/testing/embedding/test_thread.py --- a/testing/embedding/test_thread.py +++ b/testing/embedding/test_thread.py @@ -5,7 +5,7 @@ def test_first_calls_in_parallel(self): add1_cffi = self.prepare_module('add1') self.compile('thread1-test', [add1_cffi], threads=True) - for i in range(50): + for i in range(20): output = self.execute('thread1-test') assert output == ("starting\n" "preparing...\n" + From pypy.commits at gmail.com Fri Oct 26 05:23:16 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 02:23:16 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Forward-port from default Message-ID: <5bd2dd04.1c69fb81.1239a.bccc@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r95246:1d9db7971be3 Date: 2018-10-26 11:10 +0200 http://bitbucket.org/pypy/pypy/changeset/1d9db7971be3/ Log: Forward-port from default diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -49,7 +49,8 @@ # Run signal handlers if we were interrupted space.getexecutioncontext().checksignals() if microseconds >= 0: - microseconds = r_longlong(endtime - (time.time() * 1e6)) + microseconds = r_longlong((endtime - (time.time() * 1e6)) + + 0.999) # Check for negative values, since those mean block # forever if microseconds <= 0: From pypy.commits at gmail.com Fri Oct 26 05:30:53 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 02:30:53 -0700 (PDT) Subject: [pypy-commit] pypy rlock-in-rpython: Issue #2905 Message-ID: <5bd2decd.1c69fb81.d7579.89f3@mx.google.com> Author: Armin Rigo Branch: rlock-in-rpython Changeset: r95247:2cb3f4e48db6 Date: 2018-10-26 11:07 +0200 http://bitbucket.org/pypy/pypy/changeset/2cb3f4e48db6/ Log: Issue #2905 A branch to backport py3.5's RPython version of W_RLock From pypy.commits at gmail.com Fri Oct 26 05:30:55 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 02:30:55 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Don't stick these tests into some unrelated test class, particularly Message-ID: <5bd2decf.1c69fb81.d875f.c619@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r95248:a70dd232b6a3 Date: 2018-10-26 11:29 +0200 http://bitbucket.org/pypy/pypy/changeset/a70dd232b6a3/ Log: Don't stick these tests into some unrelated test class, particularly one that is skipped on some platforms diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -306,6 +306,9 @@ finally: signal.signal(signal.SIGALRM, oldalrm) + +class AppTestLockRepr(GenericTestThread): + def test_lock_repr(self): import _thread lock = _thread.allocate_lock() From pypy.commits at gmail.com Fri Oct 26 05:55:12 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 02:55:12 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: aaaaaaaaa Message-ID: <5bd2e480.1c69fb81.87246.aada@mx.google.com> Author: Armin Rigo Branch: py3.5 Changeset: r95249:78890f8b8979 Date: 2018-10-26 11:54 +0200 http://bitbucket.org/pypy/pypy/changeset/78890f8b8979/ Log: aaaaaaaaa diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -248,7 +248,7 @@ "cannot release un-acquired lock") self.rlock_count -= 1 if self.rlock_count == 0: - self.rlock_owner == 0 + self.rlock_owner = 0 self.lock.release() def is_owned_w(self, space): From pypy.commits at gmail.com Fri Oct 26 06:04:16 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 03:04:16 -0700 (PDT) Subject: [pypy-commit] pypy default: fix test Message-ID: <5bd2e6a0.1c69fb81.e1b91.3fb7@mx.google.com> Author: Armin Rigo Branch: Changeset: r95250:b01d1db30165 Date: 2018-10-26 12:01 +0200 http://bitbucket.org/pypy/pypy/changeset/b01d1db30165/ Log: fix test diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py --- a/pypy/module/cpyext/test/test_misc.py +++ b/pypy/module/cpyext/test/test_misc.py @@ -16,7 +16,7 @@ '''), ], prologue=''' static long my_flag = 0; - static int my_callback(void) { my_flag++; } + static int my_callback(void) { return ++my_flag; } ''') try: From pypy.commits at gmail.com Fri Oct 26 06:15:53 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 03:15:53 -0700 (PDT) Subject: [pypy-commit] pypy rlock-in-rpython: Backport Message-ID: <5bd2e959.1c69fb81.20a50.6ed0@mx.google.com> Author: Armin Rigo Branch: rlock-in-rpython Changeset: r95251:4874b6c33f95 Date: 2018-10-26 12:15 +0200 http://bitbucket.org/pypy/pypy/changeset/4874b6c33f95/ Log: Backport diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -36,6 +36,10 @@ _allocate_lock = thread.allocate_lock _get_ident = thread.get_ident ThreadError = thread.error +try: + _CRLock = _thread.RLock +except AttributeError: + _CRLock = None del thread @@ -120,7 +124,9 @@ acquired it. """ - return _RLock(*args, **kwargs) + if _CRLock is None or args or kwargs: + return _PyRLock(*args, **kwargs) + return _CRLock(_active) class _RLock(_Verbose): """A reentrant lock must be released by the thread that acquired it. Once a @@ -238,6 +244,8 @@ def _is_owned(self): return self.__owner == _get_ident() +_PyRLock = _RLock + def Condition(*args, **kwargs): """Factory function that returns a new condition variable object. diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py --- a/pypy/module/thread/__init__.py +++ b/pypy/module/thread/__init__.py @@ -18,6 +18,7 @@ 'allocate_lock': 'os_lock.allocate_lock', 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.Lock', + 'RLock': 'os_lock.W_RLock', # pypy only, issue #2905 '_local': 'os_local.Local', 'error': 'space.fromcache(error.Cache).w_error', } diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -8,8 +8,8 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, make_weakref_descr -from pypy.interpreter.error import oefmt -from rpython.rlib.rarithmetic import r_longlong, ovfcheck_float_to_longlong +from pypy.interpreter.error import OperationError, oefmt +from rpython.rlib.rarithmetic import r_longlong, ovfcheck, ovfcheck_float_to_longlong RPY_LOCK_FAILURE, RPY_LOCK_ACQUIRED, RPY_LOCK_INTR = range(3) @@ -53,6 +53,12 @@ break return result +def try_release(space, lock): + try: + lock.release() + except rthread.error: + raise wrap_thread_error(space, "release unlocked lock") + class Lock(W_Root): "A box around an interp-level lock object." @@ -97,10 +103,7 @@ """Release the lock, allowing another thread that is blocked waiting for the lock to acquire the lock. The lock must be in the locked state, but it needn't be locked by the same thread that unlocks it.""" - try: - self.lock.release() - except rthread.error: - raise wrap_thread_error(space, "release unlocked lock") + try_release(space, self.lock) def descr_lock_locked(self, space): """Return whether the lock is in the locked state.""" @@ -162,3 +165,149 @@ """Create a new lock object. (allocate() is an obsolete synonym.) See LockType.__doc__ for information about locks.""" return Lock(space) + +class W_RLock(W_Root): + # Does not exist in CPython 2.x. Back-ported from PyPy3. See issue #2905 + + def __init__(self, space, w_active=None): + self.rlock_count = 0 + self.rlock_owner = 0 + self.w_active = w_active # dictionary 'threading._active' + try: + self.lock = rthread.allocate_lock() + except rthread.error: + raise wrap_thread_error(space, "cannot allocate lock") + + def descr__new__(space, w_subtype, w_active=None): + self = space.allocate_instance(W_RLock, w_subtype) + W_RLock.__init__(self, space, w_active) + return self + + def descr__repr__(self, space): + w_type = space.type(self) + classname = w_type.name + if self.rlock_owner == 0: + owner = "None" + else: + owner = str(self.rlock_owner) + if self.w_active is not None: + try: + w_owner = space.getitem(self.w_active, + space.newint(self.rlock_owner)) + w_name = space.getattr(w_owner, space.newtext('name')) + owner = space.str_w(space.repr(w_name)) + except OperationError as e: + if e.async(space): + raise + return space.newtext("<%s owner=%s count=%d>" % ( + classname, owner, self.rlock_count)) + + @unwrap_spec(blocking=int) + def acquire_w(self, space, blocking=1): + """Acquire a lock, blocking or non-blocking. + + When invoked without arguments: if this thread already owns the lock, + increment the recursion level by one, and return immediately. Otherwise, + if another thread owns the lock, block until the lock is unlocked. Once + the lock is unlocked (not owned by any thread), then grab ownership, set + the recursion level to one, and return. If more than one thread is + blocked waiting until the lock is unlocked, only one at a time will be + able to grab ownership of the lock. There is no return value in this + case. + + When invoked with the blocking argument set to true, do the same thing + as when called without arguments, and return true. + + When invoked with the blocking argument set to false, do not block. If a + call without an argument would block, return false immediately; + otherwise, do the same thing as when called without arguments, and + return true. + + """ + tid = rthread.get_ident() + if tid == self.rlock_owner: + try: + self.rlock_count = ovfcheck(self.rlock_count + 1) + except OverflowError: + raise oefmt(space.w_OverflowError, + "internal lock count overflowed") + return space.w_True + + rc = self.lock.acquire(blocking != 0) + if rc: + self.rlock_owner = tid + self.rlock_count = 1 + return space.newbool(rc) + + def release_w(self, space): + """Release a lock, decrementing the recursion level. + + If after the decrement it is zero, reset the lock to unlocked (not owned + by any thread), and if any other threads are blocked waiting for the + lock to become unlocked, allow exactly one of them to proceed. If after + the decrement the recursion level is still nonzero, the lock remains + locked and owned by the calling thread. + + Only call this method when the calling thread owns the lock. A + RuntimeError is raised if this method is called when the lock is + unlocked. + + There is no return value. + + """ + if self.rlock_owner != rthread.get_ident(): + raise oefmt(space.w_RuntimeError, + "cannot release un-acquired lock") + self.rlock_count -= 1 + if self.rlock_count == 0: + self.rlock_owner = 0 + try_release(space, self.lock) + + def is_owned_w(self, space): + """For internal use by `threading.Condition`.""" + return space.newbool(self.rlock_owner == rthread.get_ident()) + + def acquire_restore_w(self, space, w_count_owner): + """For internal use by `threading.Condition`.""" + # saved_state is the value returned by release_save() + w_count, w_owner = space.unpackiterable(w_count_owner, 2) + count = space.int_w(w_count) + owner = space.int_w(w_owner) + self.lock.acquire(True) + self.rlock_count = count + self.rlock_owner = owner + + def release_save_w(self, space): + """For internal use by `threading.Condition`.""" + if self.rlock_count == 0: + raise oefmt(space.w_RuntimeError, + "cannot release un-acquired lock") + count, self.rlock_count = self.rlock_count, 0 + owner, self.rlock_owner = self.rlock_owner, 0 + try_release(space, self.lock) + return space.newtuple([space.newint(count), space.newint(owner)]) + + def descr__enter__(self, space): + self.acquire_w(space) + return self + + def descr__exit__(self, space, __args__): + self.release_w(space) + + def descr__note(self, space, __args__): + pass # compatibility with the _Verbose base class in Python + +W_RLock.typedef = TypeDef( + "thread.RLock", + __new__ = interp2app(W_RLock.descr__new__.im_func), + acquire = interp2app(W_RLock.acquire_w), + release = interp2app(W_RLock.release_w), + _is_owned = interp2app(W_RLock.is_owned_w), + _acquire_restore = interp2app(W_RLock.acquire_restore_w), + _release_save = interp2app(W_RLock.release_save_w), + __enter__ = interp2app(W_RLock.descr__enter__), + __exit__ = interp2app(W_RLock.descr__exit__), + __weakref__ = make_weakref_descr(W_RLock), + __repr__ = interp2app(W_RLock.descr__repr__), + _note = interp2app(W_RLock.descr__note), + ) diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -138,6 +138,62 @@ test_lock_again = AppTestLock.test_lock.im_func +class AppTestRLock(GenericTestThread): + """ + Tests for recursive locks. + """ + def test_reacquire(self): + import thread + lock = thread.RLock() + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + + def test_release_unacquired(self): + # Cannot release an unacquired lock + import thread + lock = thread.RLock() + raises(RuntimeError, lock.release) + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + raises(RuntimeError, lock.release) + + def test_release_save(self): + import thread + lock = thread.RLock() + raises(RuntimeError, lock._release_save) + lock.acquire() + state = lock._release_save() + lock._acquire_restore(state) + lock.release() + + def test__is_owned(self): + import thread + lock = thread.RLock() + assert lock._is_owned() is False + lock.acquire() + assert lock._is_owned() is True + lock.acquire() + assert lock._is_owned() is True + lock.release() + assert lock._is_owned() is True + lock.release() + assert lock._is_owned() is False + + def test_context_manager(self): + import thread + lock = thread.RLock() + with lock: + assert lock._is_owned() is True + + class AppTestLockSignals(GenericTestThread): pytestmark = py.test.mark.skipif("os.name != 'posix'") @@ -178,6 +234,10 @@ import thread self.acquire_retries_on_intr(thread.allocate_lock()) + def test_rlock_acquire_retries_on_intr(self): + import thread + self.acquire_retries_on_intr(thread.RLock()) + def w_alarm_interrupt(self, sig, frame): raise KeyboardInterrupt @@ -209,3 +269,20 @@ assert dt < 8.0 finally: signal.signal(signal.SIGALRM, oldalrm) + + +class AppTestLockRepr(GenericTestThread): + + def test_rlock_repr(self): + import thread + class MyThread: + name = "foobar" + actives = {thread.get_ident(): MyThread()} + rlock = thread.RLock(actives) + assert repr(rlock) == "" + rlock.acquire() + rlock.acquire() + assert repr(rlock) == "" + actives.clear() + assert repr(rlock) == "" % ( + thread.get_ident(),) From pypy.commits at gmail.com Fri Oct 26 06:21:57 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 03:21:57 -0700 (PDT) Subject: [pypy-commit] pypy rlock-in-rpython: fix Message-ID: <5bd2eac5.1c69fb81.17c0f.e99a@mx.google.com> Author: Armin Rigo Branch: rlock-in-rpython Changeset: r95252:8f94cecf5830 Date: 2018-10-26 12:21 +0200 http://bitbucket.org/pypy/pypy/changeset/8f94cecf5830/ Log: fix diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -37,7 +37,7 @@ _get_ident = thread.get_ident ThreadError = thread.error try: - _CRLock = _thread.RLock + _CRLock = thread.RLock except AttributeError: _CRLock = None del thread From pypy.commits at gmail.com Fri Oct 26 12:43:39 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 09:43:39 -0700 (PDT) Subject: [pypy-commit] pypy rlock-in-rpython: close branch, ready to merge Message-ID: <5bd3443b.1c69fb81.9c909.41a7@mx.google.com> Author: Armin Rigo Branch: rlock-in-rpython Changeset: r95254:4f83e855f001 Date: 2018-10-26 18:42 +0200 http://bitbucket.org/pypy/pypy/changeset/4f83e855f001/ Log: close branch, ready to merge From pypy.commits at gmail.com Fri Oct 26 12:43:42 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 26 Oct 2018 09:43:42 -0700 (PDT) Subject: [pypy-commit] pypy default: hg merge rlock-in-rpython Message-ID: <5bd3443e.1c69fb81.160a2.9d48@mx.google.com> Author: Armin Rigo Branch: Changeset: r95255:55e3c71a37fa Date: 2018-10-26 18:42 +0200 http://bitbucket.org/pypy/pypy/changeset/55e3c71a37fa/ Log: hg merge rlock-in-rpython diff --git a/lib-python/2.7/threading.py b/lib-python/2.7/threading.py --- a/lib-python/2.7/threading.py +++ b/lib-python/2.7/threading.py @@ -36,6 +36,10 @@ _allocate_lock = thread.allocate_lock _get_ident = thread.get_ident ThreadError = thread.error +try: + _CRLock = thread.RLock +except AttributeError: + _CRLock = None del thread @@ -120,7 +124,9 @@ acquired it. """ - return _RLock(*args, **kwargs) + if _CRLock is None or args or kwargs: + return _PyRLock(*args, **kwargs) + return _CRLock(_active) class _RLock(_Verbose): """A reentrant lock must be released by the thread that acquired it. Once a @@ -238,6 +244,8 @@ def _is_owned(self): return self.__owner == _get_ident() +_PyRLock = _RLock + def Condition(*args, **kwargs): """Factory function that returns a new condition variable object. diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py --- a/pypy/module/thread/__init__.py +++ b/pypy/module/thread/__init__.py @@ -18,6 +18,7 @@ 'allocate_lock': 'os_lock.allocate_lock', 'allocate': 'os_lock.allocate_lock', # obsolete synonym 'LockType': 'os_lock.Lock', + 'RLock': 'os_lock.W_RLock', # pypy only, issue #2905 '_local': 'os_local.Local', 'error': 'space.fromcache(error.Cache).w_error', } diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -8,8 +8,8 @@ from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, make_weakref_descr -from pypy.interpreter.error import oefmt -from rpython.rlib.rarithmetic import r_longlong, ovfcheck_float_to_longlong +from pypy.interpreter.error import OperationError, oefmt +from rpython.rlib.rarithmetic import r_longlong, ovfcheck, ovfcheck_float_to_longlong RPY_LOCK_FAILURE, RPY_LOCK_ACQUIRED, RPY_LOCK_INTR = range(3) @@ -53,6 +53,12 @@ break return result +def try_release(space, lock): + try: + lock.release() + except rthread.error: + raise wrap_thread_error(space, "release unlocked lock") + class Lock(W_Root): "A box around an interp-level lock object." @@ -97,10 +103,7 @@ """Release the lock, allowing another thread that is blocked waiting for the lock to acquire the lock. The lock must be in the locked state, but it needn't be locked by the same thread that unlocks it.""" - try: - self.lock.release() - except rthread.error: - raise wrap_thread_error(space, "release unlocked lock") + try_release(space, self.lock) def descr_lock_locked(self, space): """Return whether the lock is in the locked state.""" @@ -162,3 +165,149 @@ """Create a new lock object. (allocate() is an obsolete synonym.) See LockType.__doc__ for information about locks.""" return Lock(space) + +class W_RLock(W_Root): + # Does not exist in CPython 2.x. Back-ported from PyPy3. See issue #2905 + + def __init__(self, space, w_active=None): + self.rlock_count = 0 + self.rlock_owner = 0 + self.w_active = w_active # dictionary 'threading._active' + try: + self.lock = rthread.allocate_lock() + except rthread.error: + raise wrap_thread_error(space, "cannot allocate lock") + + def descr__new__(space, w_subtype, w_active=None): + self = space.allocate_instance(W_RLock, w_subtype) + W_RLock.__init__(self, space, w_active) + return self + + def descr__repr__(self, space): + w_type = space.type(self) + classname = w_type.name + if self.rlock_owner == 0: + owner = "None" + else: + owner = str(self.rlock_owner) + if self.w_active is not None: + try: + w_owner = space.getitem(self.w_active, + space.newint(self.rlock_owner)) + w_name = space.getattr(w_owner, space.newtext('name')) + owner = space.str_w(space.repr(w_name)) + except OperationError as e: + if e.async(space): + raise + return space.newtext("<%s owner=%s count=%d>" % ( + classname, owner, self.rlock_count)) + + @unwrap_spec(blocking=int) + def acquire_w(self, space, blocking=1): + """Acquire a lock, blocking or non-blocking. + + When invoked without arguments: if this thread already owns the lock, + increment the recursion level by one, and return immediately. Otherwise, + if another thread owns the lock, block until the lock is unlocked. Once + the lock is unlocked (not owned by any thread), then grab ownership, set + the recursion level to one, and return. If more than one thread is + blocked waiting until the lock is unlocked, only one at a time will be + able to grab ownership of the lock. There is no return value in this + case. + + When invoked with the blocking argument set to true, do the same thing + as when called without arguments, and return true. + + When invoked with the blocking argument set to false, do not block. If a + call without an argument would block, return false immediately; + otherwise, do the same thing as when called without arguments, and + return true. + + """ + tid = rthread.get_ident() + if tid == self.rlock_owner: + try: + self.rlock_count = ovfcheck(self.rlock_count + 1) + except OverflowError: + raise oefmt(space.w_OverflowError, + "internal lock count overflowed") + return space.w_True + + rc = self.lock.acquire(blocking != 0) + if rc: + self.rlock_owner = tid + self.rlock_count = 1 + return space.newbool(rc) + + def release_w(self, space): + """Release a lock, decrementing the recursion level. + + If after the decrement it is zero, reset the lock to unlocked (not owned + by any thread), and if any other threads are blocked waiting for the + lock to become unlocked, allow exactly one of them to proceed. If after + the decrement the recursion level is still nonzero, the lock remains + locked and owned by the calling thread. + + Only call this method when the calling thread owns the lock. A + RuntimeError is raised if this method is called when the lock is + unlocked. + + There is no return value. + + """ + if self.rlock_owner != rthread.get_ident(): + raise oefmt(space.w_RuntimeError, + "cannot release un-acquired lock") + self.rlock_count -= 1 + if self.rlock_count == 0: + self.rlock_owner = 0 + try_release(space, self.lock) + + def is_owned_w(self, space): + """For internal use by `threading.Condition`.""" + return space.newbool(self.rlock_owner == rthread.get_ident()) + + def acquire_restore_w(self, space, w_count_owner): + """For internal use by `threading.Condition`.""" + # saved_state is the value returned by release_save() + w_count, w_owner = space.unpackiterable(w_count_owner, 2) + count = space.int_w(w_count) + owner = space.int_w(w_owner) + self.lock.acquire(True) + self.rlock_count = count + self.rlock_owner = owner + + def release_save_w(self, space): + """For internal use by `threading.Condition`.""" + if self.rlock_count == 0: + raise oefmt(space.w_RuntimeError, + "cannot release un-acquired lock") + count, self.rlock_count = self.rlock_count, 0 + owner, self.rlock_owner = self.rlock_owner, 0 + try_release(space, self.lock) + return space.newtuple([space.newint(count), space.newint(owner)]) + + def descr__enter__(self, space): + self.acquire_w(space) + return self + + def descr__exit__(self, space, __args__): + self.release_w(space) + + def descr__note(self, space, __args__): + pass # compatibility with the _Verbose base class in Python + +W_RLock.typedef = TypeDef( + "thread.RLock", + __new__ = interp2app(W_RLock.descr__new__.im_func), + acquire = interp2app(W_RLock.acquire_w), + release = interp2app(W_RLock.release_w), + _is_owned = interp2app(W_RLock.is_owned_w), + _acquire_restore = interp2app(W_RLock.acquire_restore_w), + _release_save = interp2app(W_RLock.release_save_w), + __enter__ = interp2app(W_RLock.descr__enter__), + __exit__ = interp2app(W_RLock.descr__exit__), + __weakref__ = make_weakref_descr(W_RLock), + __repr__ = interp2app(W_RLock.descr__repr__), + _note = interp2app(W_RLock.descr__note), + ) diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -138,6 +138,62 @@ test_lock_again = AppTestLock.test_lock.im_func +class AppTestRLock(GenericTestThread): + """ + Tests for recursive locks. + """ + def test_reacquire(self): + import thread + lock = thread.RLock() + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + + def test_release_unacquired(self): + # Cannot release an unacquired lock + import thread + lock = thread.RLock() + raises(RuntimeError, lock.release) + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + raises(RuntimeError, lock.release) + + def test_release_save(self): + import thread + lock = thread.RLock() + raises(RuntimeError, lock._release_save) + lock.acquire() + state = lock._release_save() + lock._acquire_restore(state) + lock.release() + + def test__is_owned(self): + import thread + lock = thread.RLock() + assert lock._is_owned() is False + lock.acquire() + assert lock._is_owned() is True + lock.acquire() + assert lock._is_owned() is True + lock.release() + assert lock._is_owned() is True + lock.release() + assert lock._is_owned() is False + + def test_context_manager(self): + import thread + lock = thread.RLock() + with lock: + assert lock._is_owned() is True + + class AppTestLockSignals(GenericTestThread): pytestmark = py.test.mark.skipif("os.name != 'posix'") @@ -178,6 +234,10 @@ import thread self.acquire_retries_on_intr(thread.allocate_lock()) + def test_rlock_acquire_retries_on_intr(self): + import thread + self.acquire_retries_on_intr(thread.RLock()) + def w_alarm_interrupt(self, sig, frame): raise KeyboardInterrupt @@ -209,3 +269,20 @@ assert dt < 8.0 finally: signal.signal(signal.SIGALRM, oldalrm) + + +class AppTestLockRepr(GenericTestThread): + + def test_rlock_repr(self): + import thread + class MyThread: + name = "foobar" + actives = {thread.get_ident(): MyThread()} + rlock = thread.RLock(actives) + assert repr(rlock) == "" + rlock.acquire() + rlock.acquire() + assert repr(rlock) == "" + actives.clear() + assert repr(rlock) == "" % ( + thread.get_ident(),) From pypy.commits at gmail.com Sat Oct 27 15:36:15 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 27 Oct 2018 12:36:15 -0700 (PDT) Subject: [pypy-commit] pypy default: fix whatsnew Message-ID: <5bd4be2f.1c69fb81.ffd2b.9a67@mx.google.com> Author: Matti Picus Branch: Changeset: r95256:9b4df13073e1 Date: 2018-10-27 22:34 +0300 http://bitbucket.org/pypy/pypy/changeset/9b4df13073e1/ Log: fix whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -39,3 +39,15 @@ .. branch: fix-readme-typo +.. branch: avoid_shell_injection_in_shutil + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` From pypy.commits at gmail.com Sat Oct 27 15:36:18 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 27 Oct 2018 12:36:18 -0700 (PDT) Subject: [pypy-commit] pypy default: fix translation Message-ID: <5bd4be32.1c69fb81.ffd2b.9a70@mx.google.com> Author: Matti Picus Branch: Changeset: r95257:e32a30711384 Date: 2018-10-27 22:34 +0300 http://bitbucket.org/pypy/pypy/changeset/e32a30711384/ Log: fix translation diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -195,7 +195,7 @@ w_owner = space.getitem(self.w_active, space.newint(self.rlock_owner)) w_name = space.getattr(w_owner, space.newtext('name')) - owner = space.str_w(space.repr(w_name)) + owner = space.text_w(space.repr(w_name)) except OperationError as e: if e.async(space): raise From pypy.commits at gmail.com Sun Oct 28 02:52:33 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 27 Oct 2018 23:52:33 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5, ignoring backported thread lock Message-ID: <5bd55cb1.1c69fb81.98e3.aa9e@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r95258:c5417c0faf2d Date: 2018-10-28 08:48 +0200 http://bitbucket.org/pypy/pypy/changeset/c5417c0faf2d/ Log: merge default into py3.5, ignoring backported thread lock diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -39,3 +39,15 @@ .. branch: fix-readme-typo +.. branch: avoid_shell_injection_in_shutil + +Backport CPython fix for possible shell injection issue in `distutils.spawn`, +https://bugs.python.org/issue34540 + +.. branch: cffi_dlopen_unicode + +Enable use of unicode file names in `dlopen` + +.. branch: rlock-in-rpython + +Backport CPython fix for `thread.RLock` diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py --- a/pypy/module/cpyext/test/test_misc.py +++ b/pypy/module/cpyext/test/test_misc.py @@ -16,7 +16,7 @@ '''), ], prologue=''' static long my_flag = 0; - static int my_callback(void) { my_flag++; } + static int my_callback(void) { return ++my_flag; } ''') try: diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py --- a/pypy/module/pwd/interp_pwd.py +++ b/pypy/module/pwd/interp_pwd.py @@ -37,7 +37,8 @@ passwd_p = lltype.Ptr(config['passwd']) def external(name, args, result, **kwargs): - return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs) + return rffi.llexternal(name, args, result, compilation_info=eci, + releasegil=False, **kwargs) c_getpwuid = external("getpwuid", [uid_t], passwd_p) c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p) From pypy.commits at gmail.com Sun Oct 28 02:52:36 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 27 Oct 2018 23:52:36 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch Message-ID: <5bd55cb4.1c69fb81.3c9e4.60dd@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95259:aa2fc2a19658 Date: 2018-10-28 08:50 +0200 http://bitbucket.org/pypy/pypy/changeset/aa2fc2a19658/ Log: merge py3.5 into branch diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py --- a/lib_pypy/pyrepl/unix_console.py +++ b/lib_pypy/pyrepl/unix_console.py @@ -27,6 +27,12 @@ from .console import Console, Event from .unix_eventqueue import EventQueue from .trace import trace +try: + from __pypy__ import pyos_inputhook +except ImportError: + def pyos_inputhook(): + pass + class InvalidTerminal(RuntimeError): pass @@ -76,8 +82,8 @@ pass def register(self, fd, flag): self.fd = fd - def poll(self, timeout=None): - r,w,e = select.select([self.fd],[],[],timeout) + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r,w,e = select.select([self.fd],[],[]) return r POLLIN = getattr(select, "POLLIN", None) @@ -407,6 +413,7 @@ def get_event(self, block=1): while self.event_queue.empty(): while 1: # All hail Unix! + pyos_inputhook() try: self.push_char(os.read(self.input_fd, 1)) except (IOError, OSError) as err: diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -110,6 +110,7 @@ 'stack_almost_full' : 'interp_magic.stack_almost_full', 'fsencode' : 'interp_magic.fsencode', 'fsdecode' : 'interp_magic.fsdecode', + 'pyos_inputhook' : 'interp_magic.pyos_inputhook', } submodules = { diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -209,3 +209,13 @@ def revdb_stop(space): from pypy.interpreter.reverse_debugging import stop_point stop_point() + +def pyos_inputhook(space): + """Call PyOS_InputHook() from the CPython C API.""" + if not space.config.objspace.usemodules.cpyext: + return + w_modules = space.sys.get('modules') + if space.finditem_str(w_modules, 'cpyext') is None: + return # cpyext not imported yet, ignore + from pypy.module.cpyext.api import invoke_pyos_inputhook + invoke_pyos_inputhook(space) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -647,6 +647,7 @@ 'Py_FrozenFlag', 'Py_TabcheckFlag', 'Py_UnicodeFlag', 'Py_IgnoreEnvironmentFlag', 'Py_DivisionWarningFlag', 'Py_DontWriteBytecodeFlag', 'Py_NoUserSiteDirectory', '_Py_QnewFlag', 'Py_Py3kWarningFlag', 'Py_HashRandomizationFlag', '_Py_PackageContext', + 'PyOS_InputHook', 'PyMem_RawMalloc', 'PyMem_RawCalloc', 'PyMem_RawRealloc', 'PyMem_RawFree', 'PyMem_Malloc', 'PyMem_Calloc', 'PyMem_Realloc', 'PyMem_Free', @@ -1183,6 +1184,10 @@ state.C._PyPy_object_dealloc = rffi.llexternal( '_PyPy_object_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) + FUNCPTR = lltype.Ptr(lltype.FuncType([], rffi.INT)) + state.C.get_pyos_inputhook = rffi.llexternal( + '_PyPy_get_PyOS_InputHook', [], FUNCPTR, + compilation_info=eci, _nowrapper=True) def init_function(func): @@ -1789,6 +1794,12 @@ return return exec_def(space, w_mod, mod_as_pyobj) +def invoke_pyos_inputhook(space): + state = space.fromcache(State) + c_inputhook = state.C.get_pyos_inputhook() + if c_inputhook: + generic_cpy_call(space, c_inputhook) + @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h --- a/pypy/module/cpyext/include/pythonrun.h +++ b/pypy/module/cpyext/include/pythonrun.h @@ -41,6 +41,11 @@ #define Py_CompileString(str, filename, start) Py_CompileStringFlags(str, filename, start, NULL) +/* Stuff with no proper home (yet) */ +PyAPI_DATA(int) (*PyOS_InputHook)(void); +typedef int (*_pypy_pyos_inputhook)(void); +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void); + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c --- a/pypy/module/cpyext/src/missing.c +++ b/pypy/module/cpyext/src/missing.c @@ -31,3 +31,7 @@ void _Py_setfilesystemdefaultencoding(const char *enc) { Py_FileSystemDefaultEncoding = enc; } +int (*PyOS_InputHook)(void) = 0; /* only ever filled in by C extensions */ +PyAPI_FUNC(_pypy_pyos_inputhook) _PyPy_get_PyOS_InputHook(void) { + return PyOS_InputHook; +} diff --git a/pypy/module/cpyext/test/test_misc.py b/pypy/module/cpyext/test/test_misc.py new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/test_misc.py @@ -0,0 +1,35 @@ +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + + +class AppTestMisc(AppTestCpythonExtensionBase): + + def test_pyos_inputhook(self): + module = self.import_extension('foo', [ + ("set_pyos_inputhook", "METH_NOARGS", + ''' + PyOS_InputHook = &my_callback; + Py_RETURN_NONE; + '''), + ("fetch_value", "METH_NOARGS", + ''' + return PyLong_FromLong(my_flag); + '''), + ], prologue=''' + static long my_flag = 0; + static int my_callback(void) { return ++my_flag; } + ''') + + try: + import __pypy__ + except ImportError: + skip("only runs on top of pypy") + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 0 + module.set_pyos_inputhook() # <= set + assert module.fetch_value() == 0 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 1 + __pypy__.pyos_inputhook() + assert module.fetch_value() == 2 + assert module.fetch_value() == 2 diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py --- a/pypy/module/pwd/interp_pwd.py +++ b/pypy/module/pwd/interp_pwd.py @@ -37,7 +37,8 @@ passwd_p = lltype.Ptr(config['passwd']) def external(name, args, result, **kwargs): - return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs) + return rffi.llexternal(name, args, result, compilation_info=eci, + releasegil=False, **kwargs) c_getpwuid = external("getpwuid", [uid_t], passwd_p) c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p) diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -49,7 +49,8 @@ # Run signal handlers if we were interrupted space.getexecutioncontext().checksignals() if microseconds >= 0: - microseconds = r_longlong(endtime - (time.time() * 1e6)) + microseconds = r_longlong((endtime - (time.time() * 1e6)) + + 0.999) # Check for negative values, since those mean block # forever if microseconds <= 0: @@ -247,7 +248,7 @@ "cannot release un-acquired lock") self.rlock_count -= 1 if self.rlock_count == 0: - self.rlock_owner == 0 + self.rlock_owner = 0 self.lock.release() def is_owned_w(self, space): diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -306,6 +306,9 @@ finally: signal.signal(signal.SIGALRM, oldalrm) + +class AppTestLockRepr(GenericTestThread): + def test_lock_repr(self): import _thread lock = _thread.allocate_lock() From pypy.commits at gmail.com Mon Oct 29 05:53:40 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 29 Oct 2018 02:53:40 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: start drafting ddorf sprint announcement Message-ID: <5bd6d8a4.1c69fb81.4146a.eb68@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: extradoc Changeset: r5918:cf45a2a515aa Date: 2018-10-29 10:53 +0100 http://bitbucket.org/pypy/extradoc/changeset/cf45a2a515aa/ Log: start drafting ddorf sprint announcement diff --git a/sprintinfo/ddorf2019/announce.txt b/sprintinfo/ddorf2019/announce.txt new file mode 100644 --- /dev/null +++ b/sprintinfo/ddorf2019/announce.txt @@ -0,0 +1,42 @@ +Düsseldorf PyPy sprint February 4th-9th 2019 +===================================================== + +The next PyPy sprint will be held in the Computer Science department of +Heinrich-Heine Universität Düsseldorf from the 4th to the 9st of +February 2019. This is a fully public sprint, everyone is welcome to +join us. + +Topics and goals +---------------- + +To be defined + +Location +-------- + +The sprint will take place in a seminar room of the computer science +department. It is in the building 25.12 of the university campus. For +travel instructions see + + https://www.uni-duesseldorf.de/home/universitaet/weiterfuehrend/lageplan-und-anfahrt.html + + +Exact times +----------- + +Work days: starting February 4th (~morning), ending February 9th (~afternoon). +The break day will probably be Thursday. + + +Registration +------------ + +Please register by Mercurial:: + + https://bitbucket.org/pypy/extradoc/ + https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/ddorf-2019/people.txt + +or on the pypy-dev mailing list if you do not yet have check-in rights: + + http://mail.python.org/mailman/listinfo/pypy-dev + diff --git a/sprintinfo/ddorf2019/people.txt b/sprintinfo/ddorf2019/people.txt new file mode 100644 --- /dev/null +++ b/sprintinfo/ddorf2019/people.txt @@ -0,0 +1,15 @@ +People coming to the Duesseldorf sprint October 2010 +==================================================== + +People who have a ``?`` in their arrive/depart or accomodation +column are known to be coming but there are no details +available yet from them. + +============================ ============== ===================== + Name Arrive/Depart Accomodation +============================ ============== ===================== +Carl Friedrich Bolz-Tereick always there private +============================ ============== ===================== + + + From pypy.commits at gmail.com Mon Oct 29 06:06:51 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 29 Oct 2018 03:06:51 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: add some goals Message-ID: <5bd6dbbb.1c69fb81.9155b.d8f3@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: extradoc Changeset: r5919:d580da1eaee5 Date: 2018-10-29 11:06 +0100 http://bitbucket.org/pypy/extradoc/changeset/d580da1eaee5/ Log: add some goals diff --git a/sprintinfo/ddorf2019/announce.txt b/sprintinfo/ddorf2019/announce.txt --- a/sprintinfo/ddorf2019/announce.txt +++ b/sprintinfo/ddorf2019/announce.txt @@ -9,7 +9,9 @@ Topics and goals ---------------- -To be defined +- improve Python 3.6 support +- discuss benchmarking situation +- progress on utf-8 branches Location -------- From pypy.commits at gmail.com Mon Oct 29 17:49:56 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 29 Oct 2018 14:49:56 -0700 (PDT) Subject: [pypy-commit] pypy unicode-from-unicode-in-c: Move some pyunicode functions to pure C Message-ID: <5bd78084.1c69fb81.c32bf.8be8@mx.google.com> Author: Matti Picus Branch: unicode-from-unicode-in-c Changeset: r95260:d61f78777f2d Date: 2018-10-28 19:35 +0200 http://bitbucket.org/pypy/pypy/changeset/d61f78777f2d/ Log: Move some pyunicode functions to pure C diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -590,8 +590,10 @@ 'Py_FatalError', 'PyOS_snprintf', 'PyOS_vsnprintf', 'PyArg_Parse', 'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords', 'PyArg_VaParse', 'PyArg_VaParseTupleAndKeywords', '_PyArg_NoKeywords', - 'PyString_FromFormat', 'PyString_FromFormatV', - 'PyUnicode_FromFormat', 'PyUnicode_FromFormatV', + 'PyString_FromFormat', 'PyString_FromFormatV', + 'PyUnicode_FromFormat', 'PyUnicode_FromFormatV', 'PyUnicode_FromUnicode', + 'PyUnicode_FromWideChar', 'PyUnicode_AsUnicode', 'PyUnicode_GetSize', + 'PyUnicode_AsWideChar', 'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant', 'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack', '_PyArg_Parse_SizeT', '_PyArg_ParseTuple_SizeT', @@ -1185,7 +1187,7 @@ state.C.get_pyos_inputhook = rffi.llexternal( '_PyPy_get_PyOS_InputHook', [], FUNCPTR, compilation_info=eci, _nowrapper=True) - + def init_function(func): INIT_FUNCTIONS.append(func) diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h --- a/pypy/module/cpyext/include/unicodeobject.h +++ b/pypy/module/cpyext/include/unicodeobject.h @@ -9,11 +9,30 @@ PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char *format, va_list vargs); PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char *format, ...); +PyAPI_FUNC(PyObject *) PyUnicode_FromUnicode(const wchar_t *u, Py_ssize_t size); +PyAPI_FUNC(PyObject *) PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size); +PyAPI_FUNC(wchar_t*) PyUnicode_AsUnicode(PyObject *unicode); +PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize(PyObject *unicode); +PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar(PyUnicodeObject *unicode, + wchar_t *w, Py_ssize_t size); +#define Py_UNICODE_COPY(target, source, length) \ + Py_MEMCPY((target), (source), (length)*sizeof(wchar_t)) #define PyUnicode_Check(op) \ PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_UNICODE_SUBCLASS) #define PyUnicode_CheckExact(op) ((op)->ob_type == &PyUnicode_Type) +/* Fast access macros */ +#define PyUnicode_GET_SIZE(op) \ + (((PyUnicodeObject *)(op))->length) +#define PyUnicode_GET_DATA_SIZE(op) \ + (((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE)) +#define PyUnicode_AS_UNICODE(op) \ + (((PyUnicodeObject *)(op))->str) +#define PyUnicode_AS_DATA(op) \ + ((const char *)((PyUnicodeObject *)(op))->str) + + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/src/unicodeobject.c b/pypy/module/cpyext/src/unicodeobject.c --- a/pypy/module/cpyext/src/unicodeobject.c +++ b/pypy/module/cpyext/src/unicodeobject.c @@ -420,4 +420,154 @@ return ret; } +/* The empty Unicode object is shared to improve performance. */ +static PyUnicodeObject *unicode_empty = NULL; +#define _Py_RETURN_UNICODE_EMPTY() \ + do { \ + if (unicode_empty != NULL) \ + Py_INCREF(unicode_empty); \ + else { \ + unicode_empty = _PyUnicode_New(0); \ + if (unicode_empty != NULL) \ + Py_INCREF(unicode_empty); \ + } \ + return (PyObject *)unicode_empty; \ + } while (0) + +/* We allocate one more byte to make sure the string is + Ux0000 terminated; some code relies on that. +*/ + +static +PyUnicodeObject *_PyUnicode_New(Py_ssize_t length) +{ + register PyUnicodeObject *unicode; + size_t new_size; + + /* Optimization for empty strings */ + if (length == 0 && unicode_empty != NULL) { + Py_INCREF(unicode_empty); + return unicode_empty; + } + + /* Ensure we won't overflow the size. */ + if (length > ((PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) - 1)) { + return (PyUnicodeObject *)PyErr_NoMemory(); + } + + unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type); + if (unicode == NULL) + return NULL; + new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); + unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size); + + if (!unicode->str) { + PyErr_NoMemory(); + goto onError; + } + /* Initialize the first element to guard against cases where + * the caller fails before initializing str -- unicode_resize() + * reads str[0], and the Keep-Alive optimization can keep memory + * allocated for str alive across a call to unicode_dealloc(unicode). + * We don't want unicode_resize to read uninitialized memory in + * that case. + */ + unicode->str[0] = 0; + unicode->str[length] = 0; + unicode->length = length; + unicode->hash = -1; + unicode->defenc = NULL; + return unicode; + + onError: + /* XXX UNREF/NEWREF interface should be more symmetrical */ + PyObject_Del(unicode); + return NULL; +} + + +PyObject* +PyUnicode_FromUnicode(const wchar_t *u, Py_ssize_t size) +{ + /* Create a Unicode Object from the Py_UNICODE buffer u of the given size. u + * may be NULL which causes the contents to be undefined. It is the user's + * responsibility to fill in the needed data. The buffer is copied into the new + * object. If the buffer is not NULL, the return value might be a shared object. + * Therefore, modification of the resulting Unicode object is only allowed when u + * is NULL. + */ + PyUnicodeObject *unicode; + + /* Optimization for empty strings */ + if (size == 0) + _Py_RETURN_UNICODE_EMPTY(); + + + unicode = _PyUnicode_New(size); + if (!unicode) + return NULL; + + /* Copy the Unicode data into the new object */ + if (u != NULL) + Py_UNICODE_COPY(unicode->str, (wchar_t*)u, size); + + return (PyObject *)unicode; +} + +PyObject* +PyUnicode_FromWideChar(const wchar_t *char_p, Py_ssize_t length) +{ + /* + * Create a Unicode object from the wchar_t buffer w of the given size. + * Return NULL on failure. + * PyPy supposes Py_UNICODE == wchar_t + */ + return PyUnicode_FromUnicode(char_p, length); +} + +wchar_t *PyUnicode_AsUnicode(PyObject *unicode) +{ + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + goto onError; + } + return PyUnicode_AS_UNICODE(unicode); + + onError: + return NULL; +} + +Py_ssize_t PyUnicode_GetSize(PyObject *unicode) +{ + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + goto onError; + } + return PyUnicode_GET_SIZE(unicode); + + onError: + return -1; +} + +Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *unicode, + wchar_t *w, + Py_ssize_t size) +{ + if (unicode == NULL) { + PyErr_BadInternalCall(); + return -1; + } + + /* If possible, try to copy the 0-termination as well */ + if (size > PyUnicode_GET_SIZE(unicode)) + size = PyUnicode_GET_SIZE(unicode) + 1; + + memcpy(w, unicode->str, size * sizeof(wchar_t)); + if (size > PyUnicode_GET_SIZE(unicode)) + return PyUnicode_GET_SIZE(unicode); + else + return size; +} + + diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -9,6 +9,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype import sys, py from pypy.module.cpyext.unicodeobject import * +from pypy.module.cpyext.state import State class AppTestUnicodeObject(AppTestCpythonExtensionBase): def test_unicodeobject(self): @@ -116,16 +117,23 @@ """ PyObject* o = PyUnicode_FromString(""); PyUnicodeObject* u = (PyUnicodeObject*)o; + Py_ssize_t n; + wchar_t * c; - PyUnicode_GET_SIZE(u); - PyUnicode_GET_SIZE(o); + n = PyUnicode_GET_SIZE(u); + n += PyUnicode_GET_SIZE(o); - PyUnicode_GET_DATA_SIZE(u); - PyUnicode_GET_DATA_SIZE(o); + n += PyUnicode_GET_DATA_SIZE(u); + n += PyUnicode_GET_DATA_SIZE(o); - PyUnicode_AS_UNICODE(o); - PyUnicode_AS_UNICODE(u); - return o; + c = PyUnicode_AS_UNICODE(o); + c = PyUnicode_AS_UNICODE(u); + if (c == NULL) { + return o; + } + else { + return o; + } """)]) assert module.test_macro_invocations() == u'' @@ -159,11 +167,6 @@ class TestUnicode(BaseApiTest): def test_unicodeobject(self, space): - assert PyUnicode_GET_SIZE(space, space.wrap(u'sp�m')) == 4 - assert PyUnicode_GetSize(space, space.wrap(u'sp�m')) == 4 - unichar = rffi.sizeof(Py_UNICODE) - assert PyUnicode_GET_DATA_SIZE(space, space.wrap(u'sp�m')) == 4 * unichar - encoding = rffi.charp2str(PyUnicode_GetDefaultEncoding(space, )) w_default_encoding = space.call_function( space.sys.get('getdefaultencoding') @@ -184,46 +187,6 @@ rffi.free_charp(utf_8) rffi.free_charp(prev_encoding) - def test_AS(self, space): - word = space.wrap(u'spam') - array = rffi.cast(rffi.CWCHARP, PyUnicode_AS_DATA(space, word)) - array2 = PyUnicode_AS_UNICODE(space, word) - array3 = PyUnicode_AsUnicode(space, word) - for (i, char) in enumerate(space.unicode_w(word)): - assert array[i] == char - assert array2[i] == char - assert array3[i] == char - with raises_w(space, TypeError): - PyUnicode_AsUnicode(space, space.wrap('spam')) - - utf_8 = rffi.str2charp('utf-8') - encoded = PyUnicode_AsEncodedString(space, space.wrap(u'sp�m'), - utf_8, None) - assert space.unwrap(encoded) == 'sp\xef\xbf\xbdm' - encoded_obj = PyUnicode_AsEncodedObject(space, space.wrap(u'sp�m'), - utf_8, None) - assert space.eq_w(encoded, encoded_obj) - with raises_w(space, TypeError): - PyUnicode_AsEncodedString( - space, space.newtuple([1, 2, 3]), None, None) - with raises_w(space, TypeError): - PyUnicode_AsEncodedString(space, space.wrap(''), None, None) - ascii = rffi.str2charp('ascii') - replace = rffi.str2charp('replace') - encoded = PyUnicode_AsEncodedString(space, space.wrap(u'sp�m'), - ascii, replace) - assert space.unwrap(encoded) == 'sp?m' - rffi.free_charp(utf_8) - rffi.free_charp(replace) - rffi.free_charp(ascii) - - buf = rffi.unicode2wcharp(u"12345") - PyUnicode_AsWideChar(space, space.wrap(u'longword'), buf, 5) - assert rffi.wcharp2unicode(buf) == 'longw' - PyUnicode_AsWideChar(space, space.wrap(u'a'), buf, 5) - assert rffi.wcharp2unicode(buf) == 'a' - rffi.free_wcharp(buf) - def test_fromstring(self, space): s = rffi.str2charp(u'sp\x09m'.encode("utf-8")) w_res = PyUnicode_FromString(space, s) @@ -536,25 +499,6 @@ w_res = PyUnicode_Concat(space, space.wrap(u'a'), space.wrap(u'b')) assert space.unicode_w(w_res) == u'ab' - def test_copy(self, space): - w_x = space.wrap(u"abcd\u0660") - count1 = space.int_w(space.len(w_x)) - target_chunk = lltype.malloc(rffi.CWCHARP.TO, count1, flavor='raw') - - x_chunk = PyUnicode_AS_UNICODE(space, w_x) - Py_UNICODE_COPY(space, target_chunk, x_chunk, 4) - w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, 4)) - - assert space.eq_w(w_y, space.wrap(u"abcd")) - - size = PyUnicode_GET_SIZE(space, w_x) - Py_UNICODE_COPY(space, target_chunk, x_chunk, size) - w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, size)) - - assert space.eq_w(w_y, w_x) - - lltype.free(target_chunk, flavor='raw') - def test_ascii_codec(self, space): s = 'abcdefg' data = rffi.str2charp(s) diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -64,7 +64,7 @@ py_unicode = rffi.cast(PyUnicodeObject, py_obj) s = space.unicode_w(w_obj) py_unicode.c_length = len(s) - py_unicode.c_str = lltype.nullptr(rffi.CWCHARP.TO) + py_unicode.c_str = rffi.unicode2wcharp(s) py_unicode.c_hash = space.hash_w(space.newunicode(s)) py_unicode.c_defenc = lltype.nullptr(PyObject.TO) @@ -87,7 +87,7 @@ py_unicode = rffi.cast(PyUnicodeObject, py_obj) decref(space, py_unicode.c_defenc) if py_unicode.c_str: - lltype.free(py_unicode.c_str, flavor="raw") + rffi.free_wcharp(py_unicode.c_str) from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) @@ -189,82 +189,6 @@ """Get the maximum ordinal for a Unicode character.""" return runicode.UNICHR(runicode.MAXUNICODE) - at cpython_api([rffi.VOIDP], rffi.CCHARP, error=CANNOT_FAIL) -def PyUnicode_AS_DATA(space, ref): - """Return a pointer to the internal buffer of the object. o has to be a - PyUnicodeObject (not checked).""" - return rffi.cast(rffi.CCHARP, PyUnicode_AS_UNICODE(space, ref)) - - at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL) -def PyUnicode_GET_DATA_SIZE(space, w_obj): - """Return the size of the object's internal buffer in bytes. o has to be a - PyUnicodeObject (not checked).""" - return rffi.sizeof(Py_UNICODE) * PyUnicode_GET_SIZE(space, w_obj) - - at cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL) -def PyUnicode_GET_SIZE(space, w_obj): - """Return the size of the object. obj is a PyUnicodeObject (not - checked).""" - return space.len_w(w_obj) - - at cpython_api([rffi.VOIDP], rffi.CWCHARP, error=CANNOT_FAIL) -def PyUnicode_AS_UNICODE(space, ref): - """Return a pointer to the internal Py_UNICODE buffer of the object. ref - has to be a PyUnicodeObject (not checked).""" - ref_unicode = rffi.cast(PyUnicodeObject, ref) - if not ref_unicode.c_str: - # Copy unicode buffer - w_unicode = from_ref(space, rffi.cast(PyObject, ref)) - u = space.unicode_w(w_unicode) - ref_unicode.c_str = rffi.unicode2wcharp(u) - return ref_unicode.c_str - - at cpython_api([PyObject], rffi.CWCHARP) -def PyUnicode_AsUnicode(space, ref): - """Return a read-only pointer to the Unicode object's internal Py_UNICODE - buffer, NULL if unicode is not a Unicode object.""" - # Don't use PyUnicode_Check, it will realize the object :-( - w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) - if not space.issubtype_w(w_type, space.w_unicode): - raise oefmt(space.w_TypeError, "expected unicode object") - return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref)) - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def PyUnicode_GetSize(space, ref): - if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_unicode: - ref = rffi.cast(PyUnicodeObject, ref) - return ref.c_length - else: - w_obj = from_ref(space, ref) - return space.len_w(w_obj) - - at cpython_api([PyUnicodeObject, rffi.CWCHARP, Py_ssize_t], Py_ssize_t, error=-1) -def PyUnicode_AsWideChar(space, ref, buf, size): - """Copy the Unicode object contents into the wchar_t buffer w. At most - size wchar_t characters are copied (excluding a possibly trailing - 0-termination character). Return the number of wchar_t characters - copied or -1 in case of an error. Note that the resulting wchar_t - string may or may not be 0-terminated. It is the responsibility of the caller - to make sure that the wchar_t string is 0-terminated in case this is - required by the application.""" - c_str = PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref)) - c_length = ref.c_length - - # If possible, try to copy the 0-termination as well - if size > c_length: - size = c_length + 1 - - - i = 0 - while i < size: - buf[i] = c_str[i] - i += 1 - - if size > c_length: - return c_length - else: - return size - @cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def PyUnicode_GetDefaultEncoding(space): """Returns the currently active default encoding.""" @@ -327,27 +251,6 @@ return unicodeobject.encode_object(space, w_unicode, 'unicode-escape', 'strict') - at cpython_api([CONST_WSTRING, Py_ssize_t], PyObject, result_is_ll=True) -def PyUnicode_FromUnicode(space, wchar_p, length): - """Create a Unicode Object from the Py_UNICODE buffer u of the given size. u - may be NULL which causes the contents to be undefined. It is the user's - responsibility to fill in the needed data. The buffer is copied into the new - object. If the buffer is not NULL, the return value might be a shared object. - Therefore, modification of the resulting Unicode object is only allowed when u - is NULL.""" - if wchar_p: - s = rffi.wcharpsize2unicode(wchar_p, length) - return make_ref(space, space.newunicode(s)) - else: - return rffi.cast(PyObject, new_empty_unicode(space, length)) - - at cpython_api([CONST_WSTRING, Py_ssize_t], PyObject, result_is_ll=True) -def PyUnicode_FromWideChar(space, wchar_p, length): - """Create a Unicode object from the wchar_t buffer w of the given size. - Return NULL on failure.""" - # PyPy supposes Py_UNICODE == wchar_t - return PyUnicode_FromUnicode(space, wchar_p, length) - @cpython_api([PyObject, CONST_STRING], PyObject, result_is_ll=True) def _PyUnicode_AsDefaultEncodedString(space, ref, errors): # Returns a borrowed reference. @@ -679,13 +582,6 @@ """Concat two strings giving a new Unicode string.""" return space.add(w_left, w_right) - at cpython_api([rffi.CWCHARP, rffi.CWCHARP, Py_ssize_t], lltype.Void) -def Py_UNICODE_COPY(space, target, source, length): - """Roughly equivalent to memcpy() only the base size is Py_UNICODE - copies sizeof(Py_UNICODE) * length bytes from source to target""" - for i in range(0, length): - target[i] = source[i] - @cpython_api([PyObject, PyObject], PyObject) def PyUnicode_Format(space, w_format, w_args): """Return a new string object from format and args; this is analogous to From pypy.commits at gmail.com Mon Oct 29 17:49:59 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 29 Oct 2018 14:49:59 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8-py3: fix incremental encoding Message-ID: <5bd78087.1c69fb81.4146a.03eb@mx.google.com> Author: Matti Picus Branch: unicode-utf8-py3 Changeset: r95261:39a87d168b67 Date: 2018-10-29 23:48 +0200 http://bitbucket.org/pypy/pypy/changeset/39a87d168b67/ Log: fix incremental encoding diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -766,8 +766,9 @@ res, lgt, pos, bo = str_decode_utf_16_helper( data, errors, final, state.decode_error_handler, _byteorder) + # return result, consumed, byteorder for buffered incremental encoders return space.newtuple([space.newutf8(res, lgt), - space.newint(lgt), space.newint(bo)]) + space.newint(pos), space.newint(bo)]) @unwrap_spec(data='bufferstr', errors='text_or_none', byteorder=int, w_final=WrappedDefault(False)) @@ -785,8 +786,9 @@ res, lgt, pos, bo = str_decode_utf_32_helper( data, errors, final, state.decode_error_handler, _byteorder) + # return result, consumed, byteorder for buffered incremental encoders return space.newtuple([space.newutf8(res, lgt), - space.newint(lgt), space.newint(bo)]) + space.newint(pos), space.newint(bo)]) # ____________________________________________________________ # Charmap