[pypy-commit] pypy py3.5: merge default
cfbolz
pypy.commits at gmail.com
Sun Feb 17 07:40:40 EST 2019
Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: py3.5
Changeset: r96036:c87433506699
Date: 2019-02-17 12:35 +0100
http://bitbucket.org/pypy/pypy/changeset/c87433506699/
Log: merge default
diff --git a/extra_tests/cffi_tests/cffi0/test_ownlib.py b/extra_tests/cffi_tests/cffi0/test_ownlib.py
--- a/extra_tests/cffi_tests/cffi0/test_ownlib.py
+++ b/extra_tests/cffi_tests/cffi0/test_ownlib.py
@@ -352,6 +352,8 @@
def test_modify_struct_value(self):
if self.module is None:
py.test.skip("fix the auto-generation of the tiny test lib")
+ if self.Backend is CTypesBackend:
+ py.test.skip("fails with the ctypes backend on some architectures")
ffi = FFI(backend=self.Backend())
ffi.cdef("""
typedef struct {
diff --git a/extra_tests/cffi_tests/embedding/thread3-test.c b/extra_tests/cffi_tests/embedding/thread3-test.c
--- a/extra_tests/cffi_tests/embedding/thread3-test.c
+++ b/extra_tests/cffi_tests/embedding/thread3-test.c
@@ -52,5 +52,6 @@
assert(status == 0);
}
printf("done\n");
+ fflush(stdout); /* this is occasionally needed on Windows */
return 0;
}
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -5,8 +5,8 @@
from .error import CDefError, FFIError, VerificationError, VerificationMissing
from .error import PkgConfigError
-__version__ = "1.12.0"
-__version_info__ = (1, 12, 0)
+__version__ = "1.12.1"
+__version_info__ = (1, 12, 1)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,43 +8,20 @@
the same works for the other two macros. Py_DEBUG implies them,
but not the other way around.
- The implementation is messy (issue #350): on Windows, with _MSC_VER,
- we have to define Py_LIMITED_API even before including pyconfig.h.
- In that case, we guess what pyconfig.h will do to the macros above,
- and check our guess after the #include.
-
- Note that on Windows, with CPython 3.x, you need virtualenv version
- >= 16.0.0. Older versions don't copy PYTHON3.DLL. As a workaround
- you can remove the definition of Py_LIMITED_API here.
-
- See also 'py_limited_api' in cffi/setuptools_ext.py.
+ Issue #350 is still open: on Windows, the code here causes it to link
+ with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was
+ attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv
+ does not make PYTHON3.DLL available, and so the "correctly" compiled
+ version would not run inside a virtualenv. We will re-apply the fix
+ after virtualenv has been fixed for some time. For explanation, see
+ issue #355. For a workaround if you want PYTHON3.DLL and don't worry
+ about virtualenv, see issue #350. See also 'py_limited_api' in
+ setuptools_ext.py.
*/
#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
-# ifdef _MSC_VER
-# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-# define Py_LIMITED_API
-# endif
-# include <pyconfig.h>
- /* sanity-check: Py_LIMITED_API will cause crashes if any of these
- are also defined. Normally, the Python file PC/pyconfig.h does not
- cause any of these to be defined, with the exception that _DEBUG
- causes Py_DEBUG. Double-check that. */
-# ifdef Py_LIMITED_API
-# if defined(Py_DEBUG)
-# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set"
-# endif
-# if defined(Py_TRACE_REFS)
-# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set"
-# endif
-# if defined(Py_REF_DEBUG)
-# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set"
-# endif
-# endif
-# else
-# include <pyconfig.h>
-# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
-# define Py_LIMITED_API
-# endif
+# include <pyconfig.h>
+# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+# define Py_LIMITED_API
# endif
#endif
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -221,7 +221,7 @@
if (f != NULL && f != Py_None) {
PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
- "\ncompiled with cffi version: 1.12.0"
+ "\ncompiled with cffi version: 1.12.1"
"\n_cffi_backend module: ", f);
modules = PyImport_GetModuleDict();
mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,8 +81,14 @@
it doesn't so far, creating troubles. That's why we check
for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent
of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401)
+
+ On Windows, with CPython <= 3.4, it's better not to use py_limited_api
+ because virtualenv *still* doesn't copy PYTHON3.DLL on these versions.
+ For now we'll skip py_limited_api on all Windows versions to avoid an
+ inconsistent mess.
"""
- if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'):
+ if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount')
+ and sys.platform != 'win32'):
import setuptools
try:
setuptools_major_version = int(setuptools.__version__.partition('.')[0])
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
from rpython.rlib import rdynload, clibffi
from rpython.rtyper.lltypesystem import rffi
-VERSION = "1.12.0"
+VERSION = "1.12.1"
FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.12.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.12.1", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -104,7 +104,8 @@
def PyList_GET_SIZE(space, w_obj):
"""Macro form of PyList_Size() without error checking.
"""
- return space.len_w(w_obj)
+ assert isinstance(w_obj, W_ListObject)
+ return w_obj.length()
@cpython_api([PyObject], Py_ssize_t, error=-1)
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -50,50 +50,56 @@
converted to a sequence, and raises a TypeError, raise a new TypeError with
m as the message text. If the conversion otherwise, fails, reraise the
original exception"""
- if isinstance(w_obj, tupleobject.W_TupleObject):
+ if isinstance(w_obj, tupleobject.W_AbstractTupleObject):
return w_obj # CCC avoid the double conversion that occurs here
if isinstance(w_obj, W_ListObject):
- # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM
- w_obj.convert_to_cpy_strategy(space)
+ # note: we used to call w_obj.convert_to_cpy_strategy() here,
+ # but we really have to call it from PySequence_Fast_GET_ITEM()
+ # because some people never call PySequence_Fast() if they know
+ # the object is a list.
return w_obj
try:
- return W_ListObject.newlist_cpyext(space, space.listview(w_obj))
+ return tupleobject.W_TupleObject(space.fixedview(w_obj))
except OperationError as e:
if e.match(space, space.w_TypeError):
raise OperationError(space.w_TypeError, space.newtext(rffi.charp2str(m)))
raise e
-# CCC this should be written as a C macro
- at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_borrowed=True)
-def PySequence_Fast_GET_ITEM(space, w_obj, index):
+# CCC this should be written as a C macro, at least for the tuple case
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
+def PySequence_Fast_GET_ITEM(space, py_obj, index):
"""Return the ith element of o, assuming that o was returned by
PySequence_Fast(), o is not NULL, and that i is within bounds.
"""
- if isinstance(w_obj, W_ListObject):
- return w_obj.getitem(index)
- elif isinstance(w_obj, tupleobject.W_TupleObject):
- return w_obj.wrappeditems[index]
- raise oefmt(space.w_TypeError,
- "PySequence_Fast_GET_ITEM called but object is not a list or "
- "sequence")
+ py_obj = rffi.cast(PyObject, py_obj)
+ if PyTuple_Check(space, py_obj):
+ from pypy.module.cpyext.tupleobject import PyTupleObject
+ py_tuple = rffi.cast(PyTupleObject, py_obj)
+ return py_tuple.c_ob_item[index]
+ else:
+ from pypy.module.cpyext.listobject import PyList_GET_ITEM
+ w_obj = from_ref(space, py_obj)
+ return PyList_GET_ITEM(space, w_obj, index)
@cpython_api([rffi.VOIDP], Py_ssize_t, error=CANNOT_FAIL)
-def PySequence_Fast_GET_SIZE(space, w_obj):
+def PySequence_Fast_GET_SIZE(space, py_obj):
"""Returns the length of o, assuming that o was returned by
PySequence_Fast() and that o is not NULL. The size can also be
gotten by calling PySequence_Size() on o, but
PySequence_Fast_GET_SIZE() is faster because it can assume o is a list
or tuple."""
- if isinstance(w_obj, W_ListObject):
- return w_obj.length()
- elif isinstance(w_obj, tupleobject.W_TupleObject):
- return len(w_obj.wrappeditems)
- raise oefmt(space.w_TypeError,
- "PySequence_Fast_GET_SIZE called but object is not a list or "
- "sequence")
+ py_obj = rffi.cast(PyObject, py_obj)
+ if PyTuple_Check(space, py_obj):
+ from pypy.module.cpyext.tupleobject import PyTupleObject
+ py_tuple = rffi.cast(PyTupleObject, py_obj)
+ return py_tuple.c_ob_size
+ else:
+ from pypy.module.cpyext.listobject import PyList_GET_SIZE
+ w_obj = from_ref(space, py_obj)
+ return PyList_GET_SIZE(space, w_obj)
@cpython_api([rffi.VOIDP], PyObjectP)
-def PySequence_Fast_ITEMS(space, w_obj):
+def PySequence_Fast_ITEMS(space, py_obj):
"""Return the underlying array of PyObject pointers. Assumes that o was returned
by PySequence_Fast() and o is not NULL.
@@ -101,18 +107,17 @@
So, only use the underlying array pointer in contexts where the sequence
cannot change.
"""
- if isinstance(w_obj, W_ListObject):
- cpy_strategy = space.fromcache(CPyListStrategy)
- if w_obj.strategy is cpy_strategy:
- return w_obj.get_raw_items() # asserts it's a cpyext strategy
- elif isinstance(w_obj, tupleobject.W_TupleObject):
+ py_obj = rffi.cast(PyObject, py_obj)
+ if PyTuple_Check(space, py_obj):
from pypy.module.cpyext.tupleobject import PyTupleObject
- py_obj = as_pyobj(space, w_obj)
py_tuple = rffi.cast(PyTupleObject, py_obj)
return rffi.cast(PyObjectP, py_tuple.c_ob_item)
- raise oefmt(space.w_TypeError,
- "PySequence_Fast_ITEMS called but object is not the result of "
- "PySequence_Fast")
+ else:
+ from pypy.module.cpyext.listobject import get_list_storage
+ w_obj = from_ref(space, py_obj)
+ assert isinstance(w_obj, W_ListObject)
+ storage = get_list_storage(space, w_obj)
+ return rffi.cast(PyObjectP, storage._elems)
@cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
def PySequence_GetSlice(space, w_obj, start, end):
@@ -300,10 +305,6 @@
storage = self.unerase(w_list.lstorage)
return storage._length
- def get_raw_items(self, w_list):
- storage = self.unerase(w_list.lstorage)
- return storage._elems
-
def getslice(self, w_list, start, stop, step, length):
w_list.switch_to_object_strategy()
return w_list.strategy.getslice(w_list, start, stop, step, length)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -141,6 +141,7 @@
'_cffi_backend',
],
"objspace.disable_entrypoints_in_cffi": True}
+ spaceconfig["objspace.std.withspecialisedtuple"] = True
@classmethod
def preload_builtins(cls, space):
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -5,7 +5,7 @@
from pypy.module.cpyext.sequence import (
PySequence_Fast, PySequence_Contains, PySequence_Index,
PySequence_GetItem, PySequence_SetItem, PySequence_DelItem)
-from pypy.module.cpyext.pyobject import get_w_obj_and_decref
+from pypy.module.cpyext.pyobject import get_w_obj_and_decref, from_ref
from pypy.module.cpyext.state import State
import pytest
@@ -21,12 +21,14 @@
w_l = space.wrap([1, 2, 3, 4])
assert api.PySequence_Fast(w_l, "message") is w_l
- assert space.int_w(api.PySequence_Fast_GET_ITEM(w_l, 1)) == 2
+ py_result = api.PySequence_Fast_GET_ITEM(w_l, 1)
+ w_result = from_ref(space, py_result)
+ assert space.int_w(w_result) == 2
assert api.PySequence_Fast_GET_SIZE(w_l) == 4
w_set = space.wrap(set((1, 2, 3, 4)))
w_seq = api.PySequence_Fast(w_set, "message")
- assert space.type(w_seq) is space.w_list
+ assert space.type(w_seq) is space.w_tuple
assert space.len_w(w_seq) == 4
w_seq = api.PySequence_Tuple(w_set)
@@ -83,7 +85,7 @@
def test_get_slice_fast(self, space, api):
w_t = space.wrap([1, 2, 3, 4, 5])
- api.PySequence_Fast(w_t, "foo") # converts
+ api.PyList_GetItem(w_t, 0) # converts to cpy strategy
assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
@@ -215,7 +217,7 @@
class TestCPyListStrategy(BaseApiTest):
def test_getitem_setitem(self, space, api):
w_l = space.wrap([1, 2, 3, 4])
- api.PySequence_Fast(w_l, "foo") # converts
+ api.PyList_GetItem(w_l, 0) # converts to cpy strategy
assert space.int_w(space.len(w_l)) == 4
assert space.int_w(space.getitem(w_l, space.wrap(1))) == 2
assert space.int_w(space.getitem(w_l, space.wrap(0))) == 1
@@ -229,17 +231,17 @@
w = space.wrap
w_l = w([1, 2, 3, 4])
- api.PySequence_Fast(w_l, "foo") # converts
+ api.PyList_GetItem(w_l, 0) # converts to cpy strategy
space.call_method(w_l, 'insert', w(0), w(0))
assert space.int_w(space.len(w_l)) == 5
assert space.int_w(space.getitem(w_l, w(3))) == 3
- api.PySequence_Fast(w_l, "foo") # converts
+ api.PyList_GetItem(w_l, 0) # converts to cpy strategy
space.call_method(w_l, 'sort')
assert space.int_w(space.len(w_l)) == 5
assert space.int_w(space.getitem(w_l, w(0))) == 0
- api.PySequence_Fast(w_l, "foo") # converts
+ api.PyList_GetItem(w_l, 0) # converts to cpy strategy
w_t = space.wrap(space.fixedview(w_l))
assert space.int_w(space.len(w_t)) == 5
assert space.int_w(space.getitem(w_t, w(0))) == 0
@@ -247,22 +249,22 @@
assert space.int_w(space.len(w_l2)) == 5
assert space.int_w(space.getitem(w_l2, w(0))) == 0
- api.PySequence_Fast(w_l, "foo") # converts
+ api.PyList_GetItem(w_l, 0) # converts to cpy strategy
w_sum = space.add(w_l, w_l)
assert space.int_w(space.len(w_sum)) == 10
- api.PySequence_Fast(w_l, "foo") # converts
+ api.PyList_GetItem(w_l, 0) # converts to cpy strategy
w_prod = space.mul(w_l, space.wrap(2))
assert space.int_w(space.len(w_prod)) == 10
- api.PySequence_Fast(w_l, "foo") # converts
+ api.PyList_GetItem(w_l, 0) # converts to cpy strategy
w_l.inplace_mul(2)
assert space.int_w(space.len(w_l)) == 10
def test_getstorage_copy(self, space, api):
w = space.wrap
w_l = w([1, 2, 3, 4])
- api.PySequence_Fast(w_l, "foo") # converts
+ api.PyList_GetItem(w_l, 0) # converts to cpy strategy
w_l1 = w([])
space.setitem(w_l1, space.newslice(w(0), w(0), w(1)), w_l)
@@ -285,6 +287,10 @@
if (objects == NULL)
return NULL;
size = PySequence_Fast_GET_SIZE(foo);
+ for (i = 0; i < size; ++i) {
+ if (objects[i] != PySequence_Fast_GET_ITEM(foo, i))
+ return PyBool_FromLong(0);
+ }
common_type = size > 0 ? Py_TYPE(objects[0]) : NULL;
for (i = 1; i < size; ++i) {
if (Py_TYPE(objects[i]) != common_type) {
@@ -304,6 +310,9 @@
s = (1, 2, 3, 4)
assert module.test_fast_sequence(s[0:-1])
assert module.test_fast_sequence(s[::-1])
+ s = (1, 2) # specialized tuple
+ assert module.test_fast_sequence(s[0:-1])
+ assert module.test_fast_sequence(s[::-1])
s = "1234"
assert module.test_fast_sequence(s[0:-1])
assert module.test_fast_sequence(s[::-1])
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -514,7 +514,8 @@
py_type = rffi.cast(PyTypeObjectPtr, ref)
assert py_type.c_tp_alloc
- assert from_ref(space, py_type.c_tp_mro).wrappeditems is w_class.mro_w
+ w_tup = from_ref(space, py_type.c_tp_mro)
+ assert space.fixedview(w_tup) == w_class.mro_w
decref(space, ref)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -45,7 +45,8 @@
# for json decoder
def create_empty_unicode_key_dict(space):
return r_dict(unicode_eq, unicode_hash,
- force_non_null=True)
+ force_non_null=True,
+ simple_hash_eq=True)
def from_unicode_key_dict(space, d):
strategy = space.fromcache(UnicodeDictStrategy)
@@ -1176,6 +1177,7 @@
return unwrapped
def unwrap(self, wrapped):
+ assert type(wrapped) is self.space.UnicodeObjectCls
return wrapped
def is_correct_type(self, w_obj):
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -100,6 +100,33 @@
return w_item
+class W_FastUnicodeIterObject(W_AbstractSeqIterObject):
+ """Sequence iterator specialized for unicode objects."""
+
+ def __init__(self, w_seq):
+ from pypy.objspace.std.unicodeobject import W_UnicodeObject
+ W_AbstractSeqIterObject.__init__(self, w_seq)
+ assert isinstance(w_seq, W_UnicodeObject)
+ self.byteindex = 0
+
+ def descr_next(self, space):
+ from pypy.objspace.std.unicodeobject import W_UnicodeObject
+ from rpython.rlib import rutf8
+ w_seq = self.w_seq
+ if w_seq is None:
+ raise OperationError(space.w_StopIteration, space.w_None)
+ assert isinstance(w_seq, W_UnicodeObject)
+ index = self.index
+ if index == w_seq._length:
+ self.w_seq = None
+ raise OperationError(space.w_StopIteration, space.w_None)
+ start = self.byteindex
+ end = rutf8.next_codepoint_pos(w_seq._utf8, start)
+ w_res = W_UnicodeObject(w_seq._utf8[start:end], 1)
+ self.byteindex = end
+ return w_res
+
+
class W_FastTupleIterObject(W_AbstractSeqIterObject):
"""Sequence iterator specialized for tuples, accessing directly
their RPython-level list of wrapped objects.
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -214,13 +214,6 @@
storage = strategy.erase(list_f)
return W_ListObject.from_storage_and_strategy(space, storage, strategy)
- @staticmethod
- def newlist_cpyext(space, list):
- from pypy.module.cpyext.sequence import CPyListStrategy, CPyListStorage
- strategy = space.fromcache(CPyListStrategy)
- storage = strategy.erase(CPyListStorage(space, list))
- return W_ListObject.from_storage_and_strategy(space, storage, strategy)
-
def __repr__(self):
""" representation for debugging purposes """
return "%s(%s, %s)" % (self.__class__.__name__, self.strategy,
@@ -259,13 +252,6 @@
self.strategy = cpy_strategy
self.lstorage = cpy_strategy.erase(CPyListStorage(space, lst))
- def get_raw_items(self):
- from pypy.module.cpyext.sequence import CPyListStrategy
-
- cpy_strategy = self.space.fromcache(CPyListStrategy)
- assert self.strategy is cpy_strategy # should we return an error?
- return cpy_strategy.get_raw_items(self)
-
# ___________________________________________________
def init_from_list_w(self, list_w):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -26,6 +26,7 @@
from pypy.objspace.std.intobject import (
W_AbstractIntObject, W_IntObject, setup_prebuilt, wrapint)
from pypy.objspace.std.iterobject import W_AbstractSeqIterObject, W_SeqIterObject
+from pypy.objspace.std.iterobject import W_FastUnicodeIterObject
from pypy.objspace.std.listobject import W_ListObject
from pypy.objspace.std.longobject import W_LongObject, newlong
from pypy.objspace.std.memoryobject import W_MemoryView
@@ -365,6 +366,8 @@
return W_SliceObject(w_start, w_end, w_step)
def newseqiter(self, w_obj):
+ if type(w_obj) is W_UnicodeObject:
+ return W_FastUnicodeIterObject(w_obj)
return W_SeqIterObject(w_obj)
def newmemoryview(self, w_obj):
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -38,6 +38,18 @@
space.w_unicode, "__new__", space.w_unicode, w_uni)
assert w_new is w_uni
+ def test_fast_iter(self):
+ space = self.space
+ w_uni = space.newutf8(u"aä".encode("utf-8"), 2)
+ old_index_storage = w_uni._index_storage
+ w_iter = space.iter(w_uni)
+ w_char1 = w_iter.descr_next(space)
+ w_char2 = w_iter.descr_next(space)
+ assert w_uni._index_storage is old_index_storage
+ assert space.eq_w(w_char1, w_uni._getitem_result(space, 0))
+ assert space.eq_w(w_char2, w_uni._getitem_result(space, 1))
+
+
if HAS_HYPOTHESIS:
@given(strategies.text(), strategies.integers(min_value=0, max_value=10),
strategies.integers(min_value=-1, max_value=10))
diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py
--- a/rpython/rlib/rarithmetic.py
+++ b/rpython/rlib/rarithmetic.py
@@ -729,7 +729,9 @@
""" The JIT special-cases this too. """
from rpython.rtyper.lltypesystem import lltype
from rpython.rtyper.lltypesystem.lloperation import llop
- return llop.int_force_ge_zero(lltype.Signed, n)
+ n = llop.int_force_ge_zero(lltype.Signed, n)
+ assert n >= 0
+ return n
def int_c_div(x, y):
"""Return the result of the C-style 'x / y'. This differs from the
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -151,7 +151,10 @@
# The following methods are provided to be overriden in
# Utf8MatchContext. The non-utf8 implementation is provided
# by the FixedMatchContext abstract subclass, in order to use
- # the same @not_rpython safety trick as above.
+ # the same @not_rpython safety trick as above. If you get a
+ # "not_rpython" error during translation, either consider
+ # calling the methods xxx_indirect() instead of xxx(), or if
+ # applicable add the @specializectx decorator.
ZERO = 0
@not_rpython
def next(self, position):
@@ -460,8 +463,7 @@
ptr = self.start_ptr
if not self.next_char_ok(ctx, pattern, ptr, self.ppos3):
return
- assert not isinstance(ctx, AbstractMatchContext)
- self.start_ptr = ctx.next(ptr)
+ self.start_ptr = ctx.next_indirect(ptr)
return self.find_first_result(ctx, pattern)
def next_char_ok(self, ctx, pattern, ptr, ppos):
diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -19,7 +19,7 @@
from rpython.rlib.objectmodel import enforceargs, we_are_translated, specialize
from rpython.rlib.objectmodel import always_inline, dont_inline, try_inline
from rpython.rlib.rstring import StringBuilder
-from rpython.rlib import jit, types
+from rpython.rlib import jit, types, rarithmetic
from rpython.rlib.signature import signature, finishsigs
from rpython.rlib.types import char, none
from rpython.rlib.rarithmetic import r_uint
@@ -117,6 +117,12 @@
# chinese wikipedia, they're anywhere between 10% and 30% slower.
# In extreme cases (small, only chinese text), they're 40% slower
+# The following was found by hand to be more optimal than both,
+# on x86-64...
+_is_64bit = sys.maxint > 2**32
+_constant_ncp = rarithmetic.r_uint64(0xffff0000ffffffff)
+
+ at always_inline
def next_codepoint_pos(code, pos):
"""Gives the position of the next codepoint after pos.
Assumes valid utf8. 'pos' must be before the end of the string.
@@ -125,6 +131,13 @@
chr1 = ord(code[pos])
if chr1 <= 0x7F:
return pos + 1
+ if _is_64bit and not jit.we_are_jitted():
+ # optimized for Intel x86-64 by hand
+ res = pos + 1 + (
+ ((chr1 > 0xDF) << 1) +
+ rarithmetic.intmask((_constant_ncp >> (chr1 & 0x3F)) & 1))
+ assert res >= 0
+ return res
if chr1 <= 0xDF:
return pos + 2
if chr1 <= 0xEF:
@@ -162,7 +175,6 @@
ordch1 = ord(code[pos])
if ordch1 <= 0x7F or pos +1 >= lgt:
return ordch1
-
ordch2 = ord(code[pos+1])
if ordch1 <= 0xDF or pos +2 >= lgt:
# 110yyyyy 10zzzzzz -> 00000000 00000yyy yyzzzzzz
@@ -518,7 +530,7 @@
break
return storage
- at jit.dont_look_inside
+ at jit.elidable
def codepoint_position_at_index(utf8, storage, index):
""" Return byte index of a character inside utf8 encoded string, given
storage of type UTF8_INDEX_STORAGE. The index must be smaller than
@@ -546,7 +558,7 @@
pos = next_codepoint_pos(utf8, pos)
return pos
- at jit.dont_look_inside
+ at jit.elidable
def codepoint_at_index(utf8, storage, index):
""" Return codepoint of a character inside utf8 encoded string, given
storage of type UTF8_INDEX_STORAGE
@@ -564,7 +576,7 @@
bytepos = next_codepoint_pos(utf8, bytepos)
return codepoint_at_pos(utf8, bytepos)
- at jit.dont_look_inside
+ at jit.elidable
def codepoint_index_at_byte_position(utf8, storage, bytepos):
""" Return the character index for which
codepoint_position_at_index(index) == bytepos.
More information about the pypy-commit
mailing list