[pypy-commit] pypy cppyy-packaging: merge default into branch for testing
wlav
pypy.commits at gmail.com
Sat Oct 28 18:58:17 EDT 2017
Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: cppyy-packaging
Changeset: r92871:766bf4f67f39
Date: 2017-10-28 15:47 -0700
http://bitbucket.org/pypy/pypy/changeset/766bf4f67f39/
Log: merge default into branch for testing
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -71,6 +71,8 @@
^lib_pypy/.+.c$
^lib_pypy/.+.o$
^lib_pypy/.+.so$
+^lib_pypy/.+.pyd$
+^lib_pypy/Release/
^pypy/doc/discussion/.+\.html$
^include/.+\.h$
^include/.+\.inl$
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -8,60 +8,63 @@
class ArrayMeta(_CDataMeta):
def __new__(self, name, cls, typedict):
res = type.__new__(self, name, cls, typedict)
- if '_type_' in typedict:
- ffiarray = _rawffi.Array(typedict['_type_']._ffishape_)
- res._ffiarray = ffiarray
- subletter = getattr(typedict['_type_'], '_type_', None)
- if subletter == 'c':
- def getvalue(self):
- return _rawffi.charp2string(self._buffer.buffer,
- self._length_)
- def setvalue(self, val):
- # we don't want to have buffers here
- if len(val) > self._length_:
- raise ValueError("%r too long" % (val,))
- if isinstance(val, str):
- _rawffi.rawstring2charp(self._buffer.buffer, val)
- else:
- for i in range(len(val)):
- self[i] = val[i]
- if len(val) < self._length_:
- self._buffer[len(val)] = '\x00'
- res.value = property(getvalue, setvalue)
- def getraw(self):
- return _rawffi.charp2rawstring(self._buffer.buffer,
- self._length_)
+ if cls == (_CData,): # this is the Array class defined below
+ res._ffiarray = None
+ return res
+ if not hasattr(res, '_length_') or not isinstance(res._length_, int):
+ raise AttributeError(
+ "class must define a '_length_' attribute, "
+ "which must be a positive integer")
+ ffiarray = res._ffiarray = _rawffi.Array(res._type_._ffishape_)
+ subletter = getattr(res._type_, '_type_', None)
+ if subletter == 'c':
+ def getvalue(self):
+ return _rawffi.charp2string(self._buffer.buffer,
+ self._length_)
+ def setvalue(self, val):
+ # we don't want to have buffers here
+ if len(val) > self._length_:
+ raise ValueError("%r too long" % (val,))
+ if isinstance(val, str):
+ _rawffi.rawstring2charp(self._buffer.buffer, val)
+ else:
+ for i in range(len(val)):
+ self[i] = val[i]
+ if len(val) < self._length_:
+ self._buffer[len(val)] = b'\x00'
+ res.value = property(getvalue, setvalue)
- def setraw(self, buffer):
- if len(buffer) > self._length_:
- raise ValueError("%r too long" % (buffer,))
- _rawffi.rawstring2charp(self._buffer.buffer, buffer)
- res.raw = property(getraw, setraw)
- elif subletter == 'u':
- def getvalue(self):
- return _rawffi.wcharp2unicode(self._buffer.buffer,
- self._length_)
+ def getraw(self):
+ return _rawffi.charp2rawstring(self._buffer.buffer,
+ self._length_)
- def setvalue(self, val):
- # we don't want to have buffers here
- if len(val) > self._length_:
- raise ValueError("%r too long" % (val,))
- if isinstance(val, unicode):
- target = self._buffer
- else:
- target = self
- for i in range(len(val)):
- target[i] = val[i]
- if len(val) < self._length_:
- target[len(val)] = u'\x00'
- res.value = property(getvalue, setvalue)
-
- if '_length_' in typedict:
- res._ffishape_ = (ffiarray, typedict['_length_'])
- res._fficompositesize_ = res._sizeofinstances()
- else:
- res._ffiarray = None
+ def setraw(self, buffer):
+ if len(buffer) > self._length_:
+ raise ValueError("%r too long" % (buffer,))
+ _rawffi.rawstring2charp(self._buffer.buffer, buffer)
+ res.raw = property(getraw, setraw)
+ elif subletter == 'u':
+ def getvalue(self):
+ return _rawffi.wcharp2unicode(self._buffer.buffer,
+ self._length_)
+
+ def setvalue(self, val):
+ # we don't want to have buffers here
+ if len(val) > self._length_:
+ raise ValueError("%r too long" % (val,))
+ if isinstance(val, unicode):
+ target = self._buffer
+ else:
+ target = self
+ for i in range(len(val)):
+ target[i] = val[i]
+ if len(val) < self._length_:
+ target[len(val)] = u'\x00'
+ res.value = property(getvalue, setvalue)
+
+ res._ffishape_ = (ffiarray, res._length_)
+ res._fficompositesize_ = res._sizeofinstances()
return res
from_address = cdata_from_address
@@ -156,7 +159,7 @@
l = [self[i] for i in range(start, stop, step)]
letter = getattr(self._type_, '_type_', None)
if letter == 'c':
- return "".join(l)
+ return b"".join(l)
if letter == 'u':
return u"".join(l)
return l
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -176,6 +176,10 @@
def _get_buffer_value(self):
return self._buffer[0]
+ def _copy_to(self, addr):
+ target = type(self).from_address(addr)._buffer
+ target[0] = self._get_buffer_value()
+
def _to_ffi_param(self):
if self.__class__._is_pointer_like():
return self._get_buffer_value()
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -114,7 +114,9 @@
cobj = self._type_.from_param(value)
if ensure_objects(cobj) is not None:
store_reference(self, index, cobj._objects)
- self._subarray(index)[0] = cobj._get_buffer_value()
+ address = self._buffer[0]
+ address += index * sizeof(self._type_)
+ cobj._copy_to(address)
def __nonzero__(self):
return self._buffer[0] != 0
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -291,6 +291,11 @@
def _get_buffer_value(self):
return self._buffer.buffer
+ def _copy_to(self, addr):
+ from ctypes import memmove
+ origin = self._get_buffer_value()
+ memmove(addr, origin, self._fficompositesize_)
+
def _to_ffi_param(self):
return self._buffer
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -1027,21 +1027,25 @@
if '\0' in sql:
raise ValueError("the query contains a null character")
- first_word = sql.lstrip().split(" ")[0].upper()
- if first_word == "":
+
+ if sql:
+ first_word = sql.lstrip().split()[0].upper()
+ if first_word == '':
+ self._type = _STMT_TYPE_INVALID
+ if first_word == "SELECT":
+ self._type = _STMT_TYPE_SELECT
+ elif first_word == "INSERT":
+ self._type = _STMT_TYPE_INSERT
+ elif first_word == "UPDATE":
+ self._type = _STMT_TYPE_UPDATE
+ elif first_word == "DELETE":
+ self._type = _STMT_TYPE_DELETE
+ elif first_word == "REPLACE":
+ self._type = _STMT_TYPE_REPLACE
+ else:
+ self._type = _STMT_TYPE_OTHER
+ else:
self._type = _STMT_TYPE_INVALID
- elif first_word == "SELECT":
- self._type = _STMT_TYPE_SELECT
- elif first_word == "INSERT":
- self._type = _STMT_TYPE_INSERT
- elif first_word == "UPDATE":
- self._type = _STMT_TYPE_UPDATE
- elif first_word == "DELETE":
- self._type = _STMT_TYPE_DELETE
- elif first_word == "REPLACE":
- self._type = _STMT_TYPE_REPLACE
- else:
- self._type = _STMT_TYPE_OTHER
if isinstance(sql, unicode):
sql = sql.encode('utf-8')
diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py
--- a/lib_pypy/_tkinter/app.py
+++ b/lib_pypy/_tkinter/app.py
@@ -119,7 +119,7 @@
tklib.TCL_GLOBAL_ONLY)
# This is used to get the application class for Tk 4.1 and up
- argv0 = className.lower()
+ argv0 = className.lower().encode('ascii')
tklib.Tcl_SetVar(self.interp, "argv0", argv0,
tklib.TCL_GLOBAL_ONLY)
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cffi
-Version: 1.11.1
+Version: 1.11.2
Summary: Foreign Function Interface for Python calling C code.
Home-page: http://cffi.readthedocs.org
Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI
from .error import CDefError, FFIError, VerificationError, VerificationMissing
-__version__ = "1.11.1"
-__version_info__ = (1, 11, 1)
+__version__ = "1.11.2"
+__version_info__ = (1, 11, 2)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -238,9 +238,9 @@
_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
{
if (sizeof(_cffi_wchar_t) == 2)
- return _cffi_from_c_wchar_t(x);
+ return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
else
- return _cffi_from_c_wchar3216_t(x);
+ return _cffi_from_c_wchar3216_t((int)x);
}
_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
@@ -254,7 +254,7 @@
_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
{
if (sizeof(_cffi_wchar_t) == 4)
- return _cffi_from_c_wchar_t(x);
+ return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
else
return _cffi_from_c_wchar3216_t(x);
}
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -247,7 +247,7 @@
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.11.1"
+ "\ncompiled with cffi version: 1.11.2"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -119,7 +119,7 @@
To run untranslated tests, you need the Boehm garbage collector libgc.
-On recent Debian and Ubuntu (like 17.04), this is the command to install
+On recent Debian and Ubuntu (16.04 onwards), this is the command to install
all build-time dependencies::
apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \
@@ -127,7 +127,7 @@
tk-dev libgc-dev python-cffi \
liblzma-dev libncursesw5-dev # these two only needed on PyPy3
-On older Debian and Ubuntu (12.04 to 16.04)::
+On older Debian and Ubuntu (12.04-14.04)::
apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
@@ -149,12 +149,23 @@
xz-devel # For lzma on PyPy3.
(XXX plus the SLES11 version of libgdbm-dev and tk-dev)
-On Mac OS X, most of these build-time dependencies are installed alongside
+On Mac OS X::
+
+Most of these build-time dependencies are installed alongside
the Developer Tools. However, note that in order for the installation to
find them you may need to run::
xcode-select --install
+An exception is OpenSSL, which is no longer provided with the operating
+system. It can be obtained via Homebrew (with ``$ brew install openssl``),
+but it will not be available on the system path by default. The easiest
+way to enable it for building pypy is to set an environment variable::
+
+ export PKG_CONFIG_PATH=$(brew --prefix)/opt/openssl/lib/pkgconfig
+
+After setting this, translation (described next) will find the OpenSSL libs
+as expected.
Run the translation
-------------------
@@ -187,18 +198,18 @@
entire pypy interpreter. This step is currently singe threaded, and RAM
hungry. As part of this step, the chain creates a large number of C code
files and a Makefile to compile them in a
- directory controlled by the ``PYPY_USESSION_DIR`` environment variable.
+ directory controlled by the ``PYPY_USESSION_DIR`` environment variable.
2. Create an executable ``pypy-c`` by running the Makefile. This step can
- utilize all possible cores on the machine.
-3. Copy the needed binaries to the current directory.
-4. Generate c-extension modules for any cffi-based stdlib modules.
+ utilize all possible cores on the machine.
+3. Copy the needed binaries to the current directory.
+4. Generate c-extension modules for any cffi-based stdlib modules.
The resulting executable behaves mostly like a normal Python
interpreter (see :doc:`cpython_differences`), and is ready for testing, for
use as a base interpreter for a new virtualenv, or for packaging into a binary
suitable for installation on another machine running the same OS as the build
-machine.
+machine.
Note that step 4 is merely done as a convenience, any of the steps may be rerun
without rerunning the previous steps.
@@ -255,7 +266,7 @@
* PyPy 2.5.1 or earlier: normal users would see permission errors.
Installers need to run ``pypy -c "import gdbm"`` and other similar
- commands at install time; the exact list is in
+ commands at install time; the exact list is in
:source:`pypy/tool/release/package.py <package.py>`. Users
seeing a broken installation of PyPy can fix it after-the-fact if they
have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -182,6 +182,57 @@
technical difficulties.
+What about numpy, numpypy, micronumpy?
+--------------------------------------
+
+Way back in 2011, the PyPy team `started to reimplement`_ numpy in PyPy. It
+has two pieces:
+
+ * the builtin module :source:`pypy/module/micronumpy`: this is written in
+ RPython and roughly covers the content of the ``numpy.core.multiarray``
+ module. Confusingly enough, this is available in PyPy under the name
+ ``_numpypy``. It is included by default in all the official releases of
+ PyPy (but it might be dropped in the future).
+
+ * a fork_ of the official numpy repository maintained by us and informally
+ called ``numpypy``: even more confusing, the name of the repo on bitbucket
+ is ``numpy``. The main difference with the upstream numpy, is that it is
+ based on the micronumpy module written in RPython, instead of of
+ ``numpy.core.multiarray`` which is written in C.
+
+Moreover, it is also possible to install the upstream version of ``numpy``:
+its core is written in C and it runs on PyPy under the cpyext compatibility
+layer. This is what you get if you do ``pypy -m pip install numpy``.
+
+
+Should I install numpy or numpypy?
+-----------------------------------
+
+TL;DR version: you should use numpy. You can install it by doing ``pypy -m pip
+install numpy``. You might also be interested in using the experimental `PyPy
+binary wheels`_ to save compilation time.
+
+The upstream ``numpy`` is written in C, and runs under the cpyext
+compatibility layer. Nowadays, cpyext is mature enough that you can simply
+use the upstream ``numpy``, since it passes 99.9% of the test suite. At the
+moment of writing (October 2017) the main drawback of ``numpy`` is that cpyext
+is infamously slow, and thus it has worse performance compared to
+``numpypy``. However, we are actively working on improving it, as we expect to
+reach the same speed, eventually.
+
+On the other hand, ``numpypy`` is more JIT-friendly and very fast to call,
+since it is written in RPython: but it is a reimplementation, and it's hard to
+be completely compatible: over the years the project slowly matured and
+eventually it was able to call out to the LAPACK and BLAS libraries to speed
+matrix calculations, and reached around an 80% parity with the upstream
+numpy. However, 80% is far from 100%. Since cpyext/numpy compatibility is
+progressing fast, we have discontinued support for ``numpypy``.
+
+.. _`started to reimplement`: https://morepypy.blogspot.co.il/2011/05/numpy-in-pypy-status-and-roadmap.html
+.. _fork: https://bitbucket.org/pypy/numpy
+.. _`PyPy binary wheels`: https://github.com/antocuni/pypy-wheels
+
+
Is PyPy more clever than CPython about Tail Calls?
--------------------------------------------------
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -240,9 +240,12 @@
**matplotlib** https://github.com/matplotlib/matplotlib
- TODO: the tkagg backend does not work, which makes tests fail on downstream
- projects like Pandas, SciPy. It uses id(obj) as a c-pointer to obj in
- tkagg.py, which requires refactoring
+ Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of
+ matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the
+ tkagg backend can function.
+
+ TODO: the matplotlib branch passes numpy arrays by value (copying all the
+ data), this proof-of-concept needs help to become completely compliant
**wxPython** https://bitbucket.org/amauryfa/wxpython-cffi
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,8 @@
.. this is a revision shortly after release-pypy2.7-v5.9.0
.. startrev:d56dadcef996
+.. branch: cppyy-packaging
+Cleanup and improve cppyy packaging
+.. branch: docs-osx-brew-openssl
+
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.11.1"
+VERSION = "1.11.2"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.11.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.2", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -66,20 +66,17 @@
"position %d from error handler out of bounds",
newpos)
replace = space.unicode_w(w_replace)
- return replace, newpos
+ if decode:
+ return replace, newpos
+ else:
+ return replace, None, newpos
return call_errorhandler
def make_decode_errorhandler(self, space):
return self._make_errorhandler(space, True)
def make_encode_errorhandler(self, space):
- errorhandler = self._make_errorhandler(space, False)
- def encode_call_errorhandler(errors, encoding, reason, input, startpos,
- endpos):
- replace, newpos = errorhandler(errors, encoding, reason, input,
- startpos, endpos)
- return replace, None, newpos
- return encode_call_errorhandler
+ return self._make_errorhandler(space, False)
def get_unicodedata_handler(self, space):
if self.unicodedata_handler:
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -290,66 +290,87 @@
def test_random_switching(self):
from _continuation import continulet
#
+ seen = []
+ #
def t1(c1):
- return c1.switch()
+ seen.append(3)
+ res = c1.switch()
+ seen.append(6)
+ return res
+ #
def s1(c1, n):
+ seen.append(2)
assert n == 123
c2 = t1(c1)
- return c1.switch('a') + 1
+ seen.append(7)
+ res = c1.switch('a') + 1
+ seen.append(10)
+ return res
#
def s2(c2, c1):
+ seen.append(5)
res = c1.switch(c2)
+ seen.append(8)
assert res == 'a'
- return c2.switch('b') + 2
+ res = c2.switch('b') + 2
+ seen.append(12)
+ return res
#
def f():
+ seen.append(1)
c1 = continulet(s1, 123)
c2 = continulet(s2, c1)
c1.switch()
+ seen.append(4)
res = c2.switch()
+ seen.append(9)
assert res == 'b'
res = c1.switch(1000)
+ seen.append(11)
assert res == 1001
- return c2.switch(2000)
+ res = c2.switch(2000)
+ seen.append(13)
+ return res
#
res = f()
assert res == 2002
+ assert seen == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
def test_f_back(self):
import sys
from _continuation import continulet
#
- def g(c):
+ def bar(c):
c.switch(sys._getframe(0))
c.switch(sys._getframe(0).f_back)
c.switch(sys._getframe(1))
c.switch(sys._getframe(1).f_back)
- assert sys._getframe(2) is f3.f_back
+ assert sys._getframe(2) is f3_foo.f_back
c.switch(sys._getframe(2))
- def f(c):
- g(c)
+ def foo(c):
+ bar(c)
#
- c = continulet(f)
- f1 = c.switch()
- assert f1.f_code.co_name == 'g'
- f2 = c.switch()
- assert f2.f_code.co_name == 'f'
- f3 = c.switch()
- assert f3 is f2
- assert f1.f_back is f3
+ c = continulet(foo)
+ f1_bar = c.switch()
+ assert f1_bar.f_code.co_name == 'bar'
+ f2_foo = c.switch()
+ assert f2_foo.f_code.co_name == 'foo'
+ f3_foo = c.switch()
+ assert f3_foo is f2_foo
+ assert f1_bar.f_back is f3_foo
def main():
- f4 = c.switch()
- assert f4.f_code.co_name == 'main', repr(f4.f_code.co_name)
- assert f3.f_back is f1 # not running, so a loop
+ f4_main = c.switch()
+ assert f4_main.f_code.co_name == 'main'
+ assert f3_foo.f_back is f1_bar # not running, so a loop
def main2():
- f5 = c.switch()
- assert f5.f_code.co_name == 'main2', repr(f5.f_code.co_name)
- assert f3.f_back is f1 # not running, so a loop
+ f5_main2 = c.switch()
+ assert f5_main2.f_code.co_name == 'main2'
+ assert f3_foo.f_back is f1_bar # not running, so a loop
main()
main2()
res = c.switch()
assert res is None
- assert f3.f_back is None
+ assert f3_foo.f_back is None
def test_traceback_is_complete(self):
import sys
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -577,6 +577,7 @@
'PyComplex_AsCComplex', 'PyComplex_FromCComplex',
'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
+ 'PyBuffer_GetPointer', 'PyBuffer_ToContiguous', 'PyBuffer_FromContiguous',
'PyOS_getsig', 'PyOS_setsig',
'PyThread_get_thread_ident', 'PyThread_allocate_lock', 'PyThread_free_lock',
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -258,14 +258,19 @@
if w_dict is None:
return 0
if not space.isinstance_w(w_dict, space.w_dict):
- return 0
+ return 0
pos = ppos[0]
py_obj = as_pyobj(space, w_dict)
py_dict = rffi.cast(PyDictObject, py_obj)
if pos == 0:
# Store the current keys in the PyDictObject.
+ from pypy.objspace.std.listobject import W_ListObject
decref(space, py_dict.c__tmpkeys)
w_keys = space.call_method(space.w_dict, "keys", w_dict)
+ # w_keys must use the object strategy in order to keep the keys alive
+ if not isinstance(w_keys, W_ListObject):
+ return 0 # XXX should not call keys() above
+ w_keys.switch_to_object_strategy()
py_dict.c__tmpkeys = create_ref(space, w_keys)
Py_IncRef(space, py_dict.c__tmpkeys)
else:
@@ -278,10 +283,10 @@
decref(space, py_dict.c__tmpkeys)
py_dict.c__tmpkeys = lltype.nullptr(PyObject.TO)
return 0
- w_key = space.listview(w_keys)[pos]
+ w_key = space.listview(w_keys)[pos] # fast iff w_keys uses object strat
w_value = space.getitem(w_dict, w_key)
if pkey:
- pkey[0] = as_pyobj(space, w_key)
+ pkey[0] = as_pyobj(space, w_key)
if pvalue:
pvalue[0] = as_pyobj(space, w_value)
return 1
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -317,6 +317,31 @@
PyAPI_FUNC(int) PyObject_AsReadBuffer(PyObject *, const void **, Py_ssize_t *);
PyAPI_FUNC(int) PyObject_AsWriteBuffer(PyObject *, void **, Py_ssize_t *);
PyAPI_FUNC(int) PyObject_CheckReadBuffer(PyObject *);
+PyAPI_FUNC(void *) PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices);
+/* Get the memory area pointed to by the indices for the buffer given.
+ Note that view->ndim is the assumed size of indices
+*/
+
+PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, Py_buffer *view,
+ Py_ssize_t len, char fort);
+PyAPI_FUNC(int) PyBuffer_FromContiguous(Py_buffer *view, void *buf,
+ Py_ssize_t len, char fort);
+/* Copy len bytes of data from the contiguous chunk of memory
+ pointed to by buf into the buffer exported by obj. Return
+ 0 on success and return -1 and raise a PyBuffer_Error on
+ error (i.e. the object does not have a buffer interface or
+ it is not working).
+
+ If fort is 'F' and the object is multi-dimensional,
+ then the data will be copied into the array in
+ Fortran-style (first dimension varies the fastest). If
+ fort is 'C', then the data will be copied into the array
+ in C-style (last dimension varies the fastest). If fort
+ is 'A', then it does not matter and the copy will be made
+ in whatever way is more efficient.
+
+*/
+
#define PyObject_MALLOC PyObject_Malloc
#define PyObject_REALLOC PyObject_Realloc
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,7 +15,7 @@
from rpython.rlib.objectmodel import keepalive_until_here
from rpython.rtyper.annlowlevel import llhelper
from rpython.rlib import rawrefcount, jit
-from rpython.rlib.debug import fatalerror
+from rpython.rlib.debug import ll_assert, fatalerror
#________________________________________________________
@@ -243,6 +243,11 @@
py_obj = rawrefcount.from_obj(PyObject, w_obj)
if not py_obj:
py_obj = create_ref(space, w_obj, w_userdata, immortal=immortal)
+ #
+ # Try to crash here, instead of randomly, if we don't keep w_obj alive
+ ll_assert(py_obj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY,
+ "Bug in cpyext: The W_Root object was garbage-collected "
+ "while being converted to PyObject.")
return py_obj
else:
return lltype.nullptr(PyObject.TO)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -13,7 +13,7 @@
ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
readbufferproc, getbufferproc, ssizessizeobjargproc)
-from pypy.module.cpyext.pyobject import make_ref, decref, from_ref
+from pypy.module.cpyext.pyobject import make_ref, from_ref, as_pyobj
from pypy.module.cpyext.pyerrors import PyErr_Occurred
from pypy.module.cpyext.memoryobject import fill_Py_buffer
from pypy.module.cpyext.state import State
@@ -90,26 +90,29 @@
args_w = space.fixedview(w_args)
return generic_cpy_call(space, func_binary, w_self, args_w[0])
+def _get_ob_type(space, w_obj):
+ # please ensure that w_obj stays alive
+ ob_type = as_pyobj(space, space.type(w_obj))
+ return rffi.cast(PyTypeObjectPtr, ob_type)
+
def wrap_binaryfunc_l(space, w_self, w_args, func):
func_binary = rffi.cast(binaryfunc, func)
check_num_args(space, w_args, 1)
args_w = space.fixedview(w_args)
- ref = make_ref(space, w_self)
- if (not ref.c_ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
+ ob_type = _get_ob_type(space, w_self)
+ if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
return space.w_NotImplemented
- decref(space, ref)
return generic_cpy_call(space, func_binary, w_self, args_w[0])
def wrap_binaryfunc_r(space, w_self, w_args, func):
func_binary = rffi.cast(binaryfunc, func)
check_num_args(space, w_args, 1)
args_w = space.fixedview(w_args)
- ref = make_ref(space, w_self)
- if (not ref.c_ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
+ ob_type = _get_ob_type(space, w_self)
+ if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
return space.w_NotImplemented
- decref(space, ref)
return generic_cpy_call(space, func_binary, args_w[0], w_self)
def wrap_ternaryfunc(space, w_self, w_args, func):
@@ -127,11 +130,10 @@
func_ternary = rffi.cast(ternaryfunc, func)
check_num_argsv(space, w_args, 1, 2)
args_w = space.fixedview(w_args)
- ref = make_ref(space, w_self)
- if (not ref.c_ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
+ ob_type = _get_ob_type(space, w_self)
+ if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
return space.w_NotImplemented
- decref(space, ref)
arg3 = space.w_None
if len(args_w) > 1:
arg3 = args_w[1]
@@ -323,12 +325,10 @@
def wrap_getreadbuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
- py_obj = make_ref(space, w_self)
- py_type = py_obj.c_ob_type
+ py_type = _get_ob_type(space, w_self)
rbp = rffi.cast(rffi.VOIDP, 0)
if py_type.c_tp_as_buffer:
rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
- decref(space, py_obj)
with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
index = rffi.cast(Py_ssize_t, 0)
size = generic_cpy_call(space, func_target, w_self, index, ptr)
@@ -341,9 +341,7 @@
def wrap_getwritebuffer(space, w_self, w_args, func):
func_target = rffi.cast(readbufferproc, func)
- py_obj = make_ref(space, w_self)
- py_type = py_obj.c_ob_type
- decref(space, py_obj)
+ py_type = _get_ob_type(space, w_self)
rbp = rffi.cast(rffi.VOIDP, 0)
if py_type.c_tp_as_buffer:
rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
@@ -359,12 +357,10 @@
def wrap_getbuffer(space, w_self, w_args, func):
func_target = rffi.cast(getbufferproc, func)
- py_obj = make_ref(space, w_self)
- py_type = py_obj.c_ob_type
+ py_type = _get_ob_type(space, w_self)
rbp = rffi.cast(rffi.VOIDP, 0)
if py_type.c_tp_as_buffer:
rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
- decref(space, py_obj)
with lltype.scoped_alloc(Py_buffer) as pybuf:
_flags = 0
if space.len_w(w_args) > 0:
diff --git a/pypy/module/cpyext/src/abstract.c b/pypy/module/cpyext/src/abstract.c
--- a/pypy/module/cpyext/src/abstract.c
+++ b/pypy/module/cpyext/src/abstract.c
@@ -101,6 +101,163 @@
return 0;
}
+void*
+PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices)
+{
+ char* pointer;
+ int i;
+ pointer = (char *)view->buf;
+ for (i = 0; i < view->ndim; i++) {
+ pointer += view->strides[i]*indices[i];
+ if ((view->suboffsets != NULL) && (view->suboffsets[i] >= 0)) {
+ pointer = *((char**)pointer) + view->suboffsets[i];
+ }
+ }
+ return (void*)pointer;
+}
+
+void
+_Py_add_one_to_index_F(int nd, Py_ssize_t *index, const Py_ssize_t *shape)
+{
+ int k;
+
+ for (k=0; k<nd; k++) {
+ if (index[k] < shape[k]-1) {
+ index[k]++;
+ break;
+ }
+ else {
+ index[k] = 0;
+ }
+ }
+}
+
+void
+_Py_add_one_to_index_C(int nd, Py_ssize_t *index, const Py_ssize_t *shape)
+{
+ int k;
+
+ for (k=nd-1; k>=0; k--) {
+ if (index[k] < shape[k]-1) {
+ index[k]++;
+ break;
+ }
+ else {
+ index[k] = 0;
+ }
+ }
+}
+
+ /* view is not checked for consistency in either of these. It is
+ assumed that the size of the buffer is view->len in
+ view->len / view->itemsize elements.
+ */
+
+int
+PyBuffer_ToContiguous(void *buf, Py_buffer *view, Py_ssize_t len, char fort)
+{
+ int k;
+ void (*addone)(int, Py_ssize_t *, const Py_ssize_t *);
+ Py_ssize_t *indices, elements;
+ char *dest, *ptr;
+
+ if (len > view->len) {
+ len = view->len;
+ }
+
+ if (PyBuffer_IsContiguous(view, fort)) {
+ /* simplest copy is all that is needed */
+ memcpy(buf, view->buf, len);
+ return 0;
+ }
+
+ /* Otherwise a more elaborate scheme is needed */
+
+ /* view->ndim <= 64 */
+ indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*(view->ndim));
+ if (indices == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ for (k=0; k<view->ndim;k++) {
+ indices[k] = 0;
+ }
+
+ if (fort == 'F') {
+ addone = _Py_add_one_to_index_F;
+ }
+ else {
+ addone = _Py_add_one_to_index_C;
+ }
+ dest = buf;
+ /* XXX : This is not going to be the fastest code in the world
+ several optimizations are possible.
+ */
+ elements = len / view->itemsize;
+ while (elements--) {
+ ptr = PyBuffer_GetPointer(view, indices);
+ memcpy(dest, ptr, view->itemsize);
+ dest += view->itemsize;
+ addone(view->ndim, indices, view->shape);
+ }
+ PyMem_Free(indices);
+ return 0;
+}
+
+int
+PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_ssize_t len, char fort)
+{
+ int k;
+ void (*addone)(int, Py_ssize_t *, const Py_ssize_t *);
+ Py_ssize_t *indices, elements;
+ char *src, *ptr;
+
+ if (len > view->len) {
+ len = view->len;
+ }
+
+ if (PyBuffer_IsContiguous(view, fort)) {
+ /* simplest copy is all that is needed */
+ memcpy(view->buf, buf, len);
+ return 0;
+ }
+
+ /* Otherwise a more elaborate scheme is needed */
+
+ /* view->ndim <= 64 */
+ indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*(view->ndim));
+ if (indices == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ for (k=0; k<view->ndim;k++) {
+ indices[k] = 0;
+ }
+
+ if (fort == 'F') {
+ addone = _Py_add_one_to_index_F;
+ }
+ else {
+ addone = _Py_add_one_to_index_C;
+ }
+ src = buf;
+ /* XXX : This is not going to be the fastest code in the world
+ several optimizations are possible.
+ */
+ elements = len / view->itemsize;
+ while (elements--) {
+ ptr = PyBuffer_GetPointer(view, indices);
+ memcpy(ptr, src, view->itemsize);
+ src += view->itemsize;
+ addone(view->ndim, indices, view->shape);
+ }
+
+ PyMem_Free(indices);
+ return 0;
+}
+
+
+
/* Buffer C-API for Python 3.0 */
int
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -115,7 +115,36 @@
view = PyMemoryView_GET_BUFFER(memoryview);
Py_DECREF(memoryview);
return PyLong_FromLong(view->len / view->itemsize);
- """)])
+ """),
+ ("test_contiguous", "METH_O",
+ """
+ Py_buffer* view;
+ PyObject * memoryview;
+ void * buf = NULL;
+ int ret;
+ Py_ssize_t len;
+ memoryview = PyMemoryView_FromObject(args);
+ if (memoryview == NULL)
+ return NULL;
+ view = PyMemoryView_GET_BUFFER(memoryview);
+ Py_DECREF(memoryview);
+ len = view->len;
+ if (len == 0)
+ return NULL;
+ buf = malloc(len);
+ ret = PyBuffer_ToContiguous(buf, view, view->len, 'A');
+ if (ret != 0)
+ {
+ free(buf);
+ return NULL;
+ }
+ ret = PyBuffer_FromContiguous(view, buf, view->len, 'A');
+ free(buf);
+ if (ret != 0)
+ return NULL;
+ Py_RETURN_NONE;
+ """),
+ ])
module = self.import_module(name='buffer_test')
arr = module.PyMyArray(10)
ten = foo.get_len(arr)
@@ -124,6 +153,7 @@
assert ten == 10
ten = foo.test_buffer(arr)
assert ten == 10
+ foo.test_contiguous(arr)
def test_releasebuffer(self):
module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -353,12 +353,8 @@
PyObject* name = PyString_FromString("mymodule");
PyObject *obj = PyType_Type.tp_alloc(&PyType_Type, 0);
PyHeapTypeObject *type = (PyHeapTypeObject*)obj;
- if ((type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE) == 0)
- {
- PyErr_SetString(PyExc_ValueError,
- "Py_TPFLAGS_HEAPTYPE not set");
- return NULL;
- }
+ /* this is issue #2434: logic from pybind11 */
+ type->ht_type.tp_flags |= Py_TPFLAGS_HEAPTYPE;
type->ht_type.tp_name = ((PyTypeObject*)args)->tp_name;
PyType_Ready(&type->ht_type);
ret = PyObject_SetAttrString((PyObject*)&type->ht_type,
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -808,7 +808,7 @@
w_encoding)
if space.is_none(w_namespace_separator):
- namespace_separator = 0
+ namespace_separator = -1
elif space.isinstance_w(w_namespace_separator, space.w_text):
separator = space.text_w(w_namespace_separator)
if len(separator) == 0:
@@ -831,7 +831,7 @@
elif space.is_w(w_intern, space.w_None):
w_intern = None
- if namespace_separator:
+ if namespace_separator >= 0:
xmlparser = XML_ParserCreateNS(
encoding,
rffi.cast(rffi.CHAR, namespace_separator))
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -59,7 +59,7 @@
p.CharacterDataHandler = lambda s: data.append(s)
encoding = encoding_arg is None and 'utf-8' or encoding_arg
- res = p.Parse(u"<xml>\u00f6</xml>".encode(encoding), isfinal=True)
+ res = p.Parse(u"<xml>\u00f6</xml>".encode(encoding), True)
assert res == 1
assert data == [u"\u00f6"]
@@ -188,6 +188,34 @@
p.ParseFile(fake_reader)
assert fake_reader.read_count == 4
+ def test_entities(self):
+ import pyexpat
+ parser = pyexpat.ParserCreate(None, "")
+
+ def startElement(tag, attrs):
+ assert tag == 'http://www.w3.org/1999/02/22-rdf-syntax-ns#RDF'
+ assert attrs == {
+ 'http://www.w3.org/XML/1998/namespacebase':
+ 'http://www.semanticweb.org/jiba/ontologies/2017/0/test'}
+ parser.StartElementHandler = startElement
+ parser.Parse("""<?xml version="1.0"?>
+
+ <!DOCTYPE rdf:RDF [
+ <!ENTITY owl "http://www.w3.org/2002/07/owl#" >
+ <!ENTITY xsd "http://www.w3.org/2001/XMLSchema#" >
+ <!ENTITY rdfs "http://www.w3.org/2000/01/rdf-schema#" >
+ <!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#" >
+ ]>
+
+ <rdf:RDF xmlns="http://www.semanticweb.org/jiba/ontologies/2017/0/test#"
+ xml:base="http://www.semanticweb.org/jiba/ontologies/2017/0/test"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:owl="http://www.w3.org/2002/07/owl#"
+ xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ </rdf:RDF>
+ """, True)
+
class AppTestPyexpat2:
spaceconfig = dict(usemodules=['pyexpat', 'itertools', '_socket',
'time', 'struct', 'binascii'])
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
@@ -37,7 +37,7 @@
v = cffi.__version__.replace('+', '')
p = os.path.join(parent, 'doc', 'source', 'installation.rst')
content = open(p).read()
- assert ("/cffi-%s.tar.gz" % v) in content
+ assert (" package version %s:" % v) in content
def test_setup_version():
parent = os.path.dirname(os.path.dirname(cffi.__file__))
diff --git a/pypy/module/test_lib_pypy/test_sqlite3.py b/pypy/module/test_lib_pypy/test_sqlite3.py
--- a/pypy/module/test_lib_pypy/test_sqlite3.py
+++ b/pypy/module/test_lib_pypy/test_sqlite3.py
@@ -228,6 +228,14 @@
cur.execute("create table test(a)")
cur.executemany("insert into test values (?)", [[1], [2], [3]])
assert cur.lastrowid is None
+ # issue 2682
+ cur.execute('''insert
+ into test
+ values (?)
+ ''', (1, ))
+ assert cur.lastrowid is not None
+ cur.execute('''insert\t into test values (?) ''', (1, ))
+ assert cur.lastrowid is not None
def test_authorizer_bad_value(self, con):
def authorizer_cb(action, arg1, arg2, dbname, source):
diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py
--- a/pypy/module/unicodedata/test/test_hyp.py
+++ b/pypy/module/unicodedata/test/test_hyp.py
@@ -40,6 +40,7 @@
@pytest.mark.parametrize('NF1, NF2, NF3', compositions)
@example(s=u'---\uafb8\u11a7---') # issue 2289
+ at example(s=u'\ufacf')
@settings(max_examples=1000)
@given(s=st.text())
def test_composition(s, space, NF1, NF2, NF3):
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -633,13 +633,16 @@
def _startswith(self, space, value, w_prefix, start, end):
prefix = self._op_val(space, w_prefix)
- if start > len(value):
- return self._starts_ends_overflow(prefix)
+ if self._starts_ends_unicode: # bug-to-bug compat
+ if len(prefix) == 0:
+ return True
+ else:
+ if start > len(value):
+ return False
return startswith(value, prefix, start, end)
- def _starts_ends_overflow(self, prefix):
- return False # bug-to-bug compat: this is for strings and
- # bytearrays, but overridden for unicodes
+ _starts_ends_unicode = False # bug-to-bug compat: this is for strings and
+ # bytearrays, but overridden for unicodes
def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
value, start, end, _ = self._convert_idx_params(space, w_start, w_end)
@@ -656,8 +659,12 @@
def _endswith(self, space, value, w_prefix, start, end):
prefix = self._op_val(space, w_prefix)
- if start > len(value):
- return self._starts_ends_overflow(prefix)
+ if self._starts_ends_unicode: # bug-to-bug compat
+ if len(prefix) == 0:
+ return True
+ else:
+ if start > len(value):
+ return False
return endswith(value, prefix, start, end)
def _strip(self, space, w_chars, left, right, name='strip'):
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -1,3 +1,6 @@
+from pypy.interpreter.error import OperationError
+
+
class TestW_BytesObject:
def teardown_method(self, method):
@@ -86,6 +89,78 @@
w_bytes = self.space.newbytes('abcd')
assert self.space.listview_bytes(w_bytes) == list("abcd")
+
+try:
+ from hypothesis import given, strategies
+except ImportError:
+ pass
+else:
+ @given(u=strategies.binary(),
+ start=strategies.integers(min_value=0, max_value=10),
+ len1=strategies.integers(min_value=-1, max_value=10))
+ def test_hypo_index_find(u, start, len1, space):
+ if start + len1 < 0:
+ return # skip this case
+ v = u[start : start + len1]
+ w_u = space.wrap(u)
+ w_v = space.wrap(v)
+ expected = u.find(v, start, start + len1)
+ try:
+ w_index = space.call_method(w_u, 'index', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ assert expected == -1
+ else:
+ assert space.int_w(w_index) == expected >= 0
+
+ w_index = space.call_method(w_u, 'find', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert space.int_w(w_index) == expected
+
+ rexpected = u.rfind(v, start, start + len1)
+ try:
+ w_index = space.call_method(w_u, 'rindex', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ assert rexpected == -1
+ else:
+ assert space.int_w(w_index) == rexpected >= 0
+
+ w_index = space.call_method(w_u, 'rfind', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert space.int_w(w_index) == rexpected
+
+ expected = u.startswith(v, start)
+ w_res = space.call_method(w_u, 'startswith', w_v,
+ space.newint(start))
+ assert w_res is space.newbool(expected)
+
+ expected = u.startswith(v, start, start + len1)
+ w_res = space.call_method(w_u, 'startswith', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert w_res is space.newbool(expected)
+
+ expected = u.endswith(v, start)
+ w_res = space.call_method(w_u, 'endswith', w_v,
+ space.newint(start))
+ assert w_res is space.newbool(expected)
+
+ expected = u.endswith(v, start, start + len1)
+ w_res = space.call_method(w_u, 'endswith', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert w_res is space.newbool(expected)
+
+
class AppTestBytesObject:
def test_format_wrongchar(self):
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -1,5 +1,6 @@
import py
import sys
+from pypy.interpreter.error import OperationError
class TestUnicodeObject:
@@ -33,6 +34,77 @@
assert w_new is w_uni
+try:
+ from hypothesis import given, strategies
+except ImportError:
+ pass
+else:
+ @given(u=strategies.text(),
+ start=strategies.integers(min_value=0, max_value=10),
+ len1=strategies.integers(min_value=-1, max_value=10))
+ def test_hypo_index_find(u, start, len1, space):
+ if start + len1 < 0:
+ return # skip this case
+ v = u[start : start + len1]
+ w_u = space.wrap(u)
+ w_v = space.wrap(v)
+ expected = u.find(v, start, start + len1)
+ try:
+ w_index = space.call_method(w_u, 'index', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ assert expected == -1
+ else:
+ assert space.int_w(w_index) == expected >= 0
+
+ w_index = space.call_method(w_u, 'find', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert space.int_w(w_index) == expected
+
+ rexpected = u.rfind(v, start, start + len1)
+ try:
+ w_index = space.call_method(w_u, 'rindex', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ assert rexpected == -1
+ else:
+ assert space.int_w(w_index) == rexpected >= 0
+
+ w_index = space.call_method(w_u, 'rfind', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert space.int_w(w_index) == rexpected
+
+ expected = u.startswith(v, start)
+ w_res = space.call_method(w_u, 'startswith', w_v,
+ space.newint(start))
+ assert w_res is space.newbool(expected)
+
+ expected = u.startswith(v, start, start + len1)
+ w_res = space.call_method(w_u, 'startswith', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert w_res is space.newbool(expected)
+
+ expected = u.endswith(v, start)
+ w_res = space.call_method(w_u, 'endswith', w_v,
+ space.newint(start))
+ assert w_res is space.newbool(expected)
+
+ expected = u.endswith(v, start, start + len1)
+ w_res = space.call_method(w_u, 'endswith', w_v,
+ space.newint(start),
+ space.newint(start + len1))
+ assert w_res is space.newbool(expected)
+
+
class AppTestUnicodeStringStdOnly:
def test_compares(self):
assert u'a' == 'a'
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -419,8 +419,7 @@
cased = True
return space.newbool(cased)
- def _starts_ends_overflow(self, prefix):
- return len(prefix) == 0
+ _starts_ends_unicode = True
def wrapunicode(space, uni):
diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst
--- a/rpython/doc/jit/optimizer.rst
+++ b/rpython/doc/jit/optimizer.rst
@@ -42,10 +42,9 @@
There are better ways to compute the sum from ``[0..100]``, but it gives a better intuition on how
traces are constructed than ``sum(range(101))``.
Note that the trace syntax is the one used in the test suite. It is also very
-similar to traces printed at runtime by PYPYLOG_. The first line gives the input variables, the
-second line is a ``label`` operation, the last one is the backwards ``jump`` operation.
-
-.. _PYPYLOG: logging.html
+similar to traces printed at runtime by :doc:`PYPYLOG <../logging>`. The first
+line gives the input variables, the second line is a ``label`` operation, the
+last one is the backwards ``jump`` operation.
These instructions mentioned earlier are special:
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -305,6 +305,10 @@
# Transform into INT_ADD. The following guard will be killed
# by optimize_GUARD_NO_OVERFLOW; if we see instead an
# optimize_GUARD_OVERFLOW, then InvalidLoop.
+
+ # NB: this case also takes care of int_add_ovf with 0 as on of the
+ # arguments: the result will be bounded, and then the optimization
+ # for int_add with 0 as argument will remove the op.
op = self.replace_op_with(op, rop.INT_ADD)
return self.emit(op)
@@ -325,6 +329,7 @@
return None
resbound = b0.sub_bound(b1)
if resbound.bounded():
+ # this case takes care of int_sub_ovf(x, 0) as well
op = self.replace_op_with(op, rop.INT_SUB)
return self.emit(op)
@@ -342,6 +347,7 @@
b2 = self.getintbound(op.getarg(1))
resbound = b1.mul_bound(b2)
if resbound.bounded():
+ # this case also takes care of multiplication with 0 and 1
op = self.replace_op_with(op, rop.INT_MUL)
return self.emit(op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -1962,6 +1962,55 @@
"""
self.optimize_loop(ops, expected)
+ ops = """
+ [i0]
+ i1 = int_mul_ovf(0, i0)
+ guard_no_overflow() []
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(0)
+ """
+ self.optimize_loop(ops, expected)
+
+ ops = """
+ [i0]
+ i1 = int_mul_ovf(i0, 0)
+ guard_no_overflow() []
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(0)
+ """
+ self.optimize_loop(ops, expected)
+
+ ops = """
+ [i0]
+ i1 = int_mul_ovf(1, i0)
+ guard_no_overflow() []
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(i0)
+ """
+ self.optimize_loop(ops, expected)
+
+ ops = """
+ [i0]
+ i1 = int_mul_ovf(i0, 1)
+ guard_no_overflow() []
+ jump(i1)
+ """
+ expected = """
+ [i0]
+ jump(i0)
+ """
+ self.optimize_loop(ops, expected)
+
+
def test_fold_constant_partial_ops_float(self):
ops = """
[f0]
diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -877,32 +877,31 @@
ch = ord(s[pos])
pos += 1
ch2 = 0
- if 0xD800 <= ch < 0xDC00:
- if not allow_surrogates:
- ru, rs, pos = errorhandler(errors, public_encoding_name,
- 'surrogates not allowed',
- s, pos-1, pos)
- if rs is not None:
- # py3k only
- if len(rs) % 4 != 0:
- errorhandler('strict', public_encoding_name,
- 'surrogates not allowed',
- s, pos-1, pos)
- result.append(rs)
- continue
- for ch in ru:
- if ord(ch) < 0xD800:
- _STORECHAR32(result, ord(ch), byteorder)
- else:
- errorhandler('strict', public_encoding_name,
- 'surrogates not allowed',
- s, pos-1, pos)
+ if not allow_surrogates and 0xD800 <= ch < 0xE000:
+ ru, rs, pos = errorhandler(errors, public_encoding_name,
+ 'surrogates not allowed',
+ s, pos-1, pos)
+ if rs is not None:
+ # py3k only
+ if len(rs) % 4 != 0:
+ errorhandler('strict', public_encoding_name,
+ 'surrogates not allowed',
+ s, pos-1, pos)
+ result.append(rs)
continue
- elif MAXUNICODE < 65536 and pos < size:
- ch2 = ord(s[pos])
- if 0xDC00 <= ch2 < 0xE000:
- ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
- pos += 1
+ for ch in ru:
+ if ord(ch) < 0xD800:
+ _STORECHAR32(result, ord(ch), byteorder)
+ else:
+ errorhandler('strict', public_encoding_name,
+ 'surrogates not allowed',
+ s, pos-1, pos)
+ continue
+ if 0xD800 <= ch < 0xDC00 and MAXUNICODE < 65536 and pos < size:
+ ch2 = ord(s[pos])
+ if 0xDC00 <= ch2 < 0xE000:
+ ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
+ pos += 1
_STORECHAR32(result, ch, byteorder)
return result.build()
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -2,6 +2,7 @@
import py
import sys, random
+import struct
from rpython.rlib import runicode
from hypothesis import given, settings, strategies
@@ -266,11 +267,12 @@
assert replace_with(u'rep', None) == '\x00<\x00r\x00e\x00p\x00>'
assert replace_with(None, '\xca\xfe') == '\x00<\xca\xfe\x00>'
- def test_utf32_surrogates(self):
+ @py.test.mark.parametrize('unich',[u"\ud800", u"\udc80"])
+ def test_utf32_surrogates(self, unich):
assert runicode.unicode_encode_utf_32_be(
- u"\ud800", 1, None) == '\x00\x00\xd8\x00'
+ unich, 1, None) == struct.pack('>i', ord(unich))
py.test.raises(UnicodeEncodeError, runicode.unicode_encode_utf_32_be,
- u"\ud800", 1, None, allow_surrogates=False)
+ unich, 1, None, allow_surrogates=False)
def replace_with(ru, rs):
def errorhandler(errors, enc, msg, u, startingpos, endingpos):
if errors == 'strict':
@@ -278,7 +280,7 @@
endingpos, msg)
return ru, rs, endingpos
return runicode.unicode_encode_utf_32_be(
- u"<\ud800>", 3, None,
+ u"<%s>" % unich, 3, None,
errorhandler, allow_surrogates=False)
assert replace_with(u'rep', None) == u'<rep>'.encode('utf-32-be')
assert replace_with(None, '\xca\xfe\xca\xfe') == '\x00\x00\x00<\xca\xfe\xca\xfe\x00\x00\x00>'
@@ -432,7 +434,7 @@
assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'ignore',
final=True) == (u'aaaabbbb', len(seq) + 8))
assert (self.decoder(seq, len(seq), 'custom', final=True,
- errorhandler=self.custom_replace) ==
+ errorhandler=self.custom_replace) ==
(FOO * len(seq), len(seq)))
assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8, 'custom',
final=True, errorhandler=self.custom_replace) ==
@@ -628,7 +630,7 @@
msg='invalid continuation byte')
assert self.decoder(seq, len(seq), 'replace', final=True
) == (res, len(seq))
- assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8,
+ assert (self.decoder('aaaa' + seq + 'bbbb', len(seq) + 8,
'replace', final=True) ==
(u'aaaa' + res + u'bbbb', len(seq) + 8))
res = res.replace(FFFD, u'')
More information about the pypy-commit
mailing list