[pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch
mattip
pypy.commits at gmail.com
Mon Jan 14 23:43:21 EST 2019
Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8-py3
Changeset: r95633:19dbd394fcf6
Date: 2019-01-15 06:39 +0200
http://bitbucket.org/pypy/pypy/changeset/19dbd394fcf6/
Log: merge py3.5 into branch
diff --git a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
--- a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
+++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
@@ -325,16 +325,22 @@
a = array.array('H', [10000, 20000, 30000])
c = ffi.from_buffer(a)
assert ffi.typeof(c) is ffi.typeof("char[]")
+ assert len(c) == 6
ffi.cast("unsigned short *", c)[1] += 500
assert list(a) == [10000, 20500, 30000]
- assert c == ffi.from_buffer(a, True)
+ assert c == ffi.from_buffer("char[]", a, True)
assert c == ffi.from_buffer(a, require_writable=True)
#
+ c = ffi.from_buffer("unsigned short[]", a)
+ assert len(c) == 3
+ assert c[1] == 20500
+ #
p = ffi.from_buffer(b"abcd")
assert p[2] == b"c"
#
- assert p == ffi.from_buffer(b"abcd", False)
- py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True)
+ assert p == ffi.from_buffer(b"abcd", require_writable=False)
+ py.test.raises((TypeError, BufferError), ffi.from_buffer,
+ "char[]", b"abcd", True)
py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
require_writable=True)
diff --git a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py
--- a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py
+++ b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py
@@ -239,19 +239,31 @@
def test_ffi_from_buffer():
import array
ffi = _cffi1_backend.FFI()
- a = array.array('H', [10000, 20000, 30000])
+ a = array.array('H', [10000, 20000, 30000, 40000])
c = ffi.from_buffer(a)
assert ffi.typeof(c) is ffi.typeof("char[]")
+ assert len(c) == 8
ffi.cast("unsigned short *", c)[1] += 500
- assert list(a) == [10000, 20500, 30000]
- assert c == ffi.from_buffer(a, True)
+ assert list(a) == [10000, 20500, 30000, 40000]
+ py.test.raises(TypeError, ffi.from_buffer, a, True)
+ assert c == ffi.from_buffer("char[]", a, True)
assert c == ffi.from_buffer(a, require_writable=True)
#
+ c = ffi.from_buffer("unsigned short[]", a)
+ assert len(c) == 4
+ assert c[1] == 20500
+ #
+ c = ffi.from_buffer("unsigned short[2][2]", a)
+ assert len(c) == 2
+ assert len(c[0]) == 2
+ assert c[0][1] == 20500
+ #
p = ffi.from_buffer(b"abcd")
assert p[2] == b"c"
#
- assert p == ffi.from_buffer(b"abcd", False)
- py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True)
+ assert p == ffi.from_buffer(b"abcd", require_writable=False)
+ py.test.raises((TypeError, BufferError), ffi.from_buffer,
+ "char[]", b"abcd", True)
py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
require_writable=True)
diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
--- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1676,24 +1676,6 @@
py.test.raises(TypeError, len, q.a)
py.test.raises(TypeError, list, q.a)
- def test_from_buffer(self):
- import array
- a = array.array('H', [10000, 20000, 30000])
- c = ffi.from_buffer(a)
- assert ffi.typeof(c) is ffi.typeof("char[]")
- ffi.cast("unsigned short *", c)[1] += 500
- assert list(a) == [10000, 20500, 30000]
- assert c == ffi.from_buffer(a, True)
- assert c == ffi.from_buffer(a, require_writable=True)
- #
- p = ffi.from_buffer(b"abcd")
- assert p[2] == b"c"
- #
- assert p == ffi.from_buffer(b"abcd", False)
- py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True)
- py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
- require_writable=True)
-
def test_all_primitives(self):
assert set(PRIMITIVE_TO_INDEX) == set([
"char",
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -16,6 +16,8 @@
# Python 3.x
basestring = str
+_unspecified = object()
+
class FFI(object):
@@ -341,15 +343,22 @@
# """
# note that 'buffer' is a type, set on this instance by __init__
- def from_buffer(self, python_buffer, require_writable=False):
- """Return a <cdata 'char[]'> that points to the data of the
+ def from_buffer(self, cdecl, python_buffer=_unspecified,
+ require_writable=False):
+ """Return a cdata of the given type pointing to the data of the
given Python object, which must support the buffer interface.
Note that this is not meant to be used on the built-in types
str or unicode (you can build 'char[]' arrays explicitly)
but only on objects containing large quantities of raw data
in some other format, like 'array.array' or numpy arrays.
+
+ The first argument is optional and default to 'char[]'.
"""
- return self._backend.from_buffer(self.BCharA, python_buffer,
+ if python_buffer is _unspecified:
+ cdecl, python_buffer = self.BCharA, cdecl
+ elif isinstance(cdecl, basestring):
+ cdecl = self._typeof(cdecl)
+ return self._backend.from_buffer(cdecl, python_buffer,
require_writable)
def memmove(self, dest, src, n):
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -13,7 +13,6 @@
assert b.build() == s
b.append("123")
assert b.build() == s + "123"
- assert type(b.build()) is str
def test_preallocate(self):
from __pypy__.builders import StringBuilder
@@ -22,7 +21,6 @@
b.append("123")
s = b.build()
assert s == "abc123"
- assert type(s) is str
def test_append_slice(self):
from __pypy__.builders import StringBuilder
@@ -45,8 +43,3 @@
assert len(b) == 16
assert s == b"abc123you and me"
assert b.build() == s
-
- def test_encode(self):
- from __pypy__.builders import UnicodeBuilder
- b = UnicodeBuilder()
- raises(UnicodeDecodeError, b.append, b'\xc0')
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -478,8 +478,9 @@
def enter_exit(self, exit_now):
raise oefmt(self.space.w_ValueError,
- "only 'cdata' object from ffi.new(), ffi.gc() or ffi.from_buffer() "
- "can be used with the 'with' keyword or ffi.release()")
+ "only 'cdata' object from ffi.new(), ffi.gc(), ffi.from_buffer() "
+ "or ffi.new_allocator()() can be used with the 'with' keyword or "
+ "ffi.release()")
def descr_enter(self):
self.enter_exit(False)
@@ -657,24 +658,28 @@
class W_CDataFromBuffer(W_CData):
_attrs_ = ['buf', 'length', 'w_keepalive']
- _immutable_fields_ = ['buf', 'length', 'w_keepalive']
+ _immutable_fields_ = ['buf', 'length']
- def __init__(self, space, cdata, ctype, buf, w_object):
+ def __init__(self, space, cdata, length, ctype, buf, w_object):
W_CData.__init__(self, space, cdata, ctype)
self.buf = buf
- self.length = buf.getlength()
+ self.length = length
self.w_keepalive = w_object
def get_array_length(self):
return self.length
def _repr_extra(self):
- w_repr = self.space.repr(self.w_keepalive)
- return "buffer len %d from '%s' object" % (
- self.length, self.space.type(self.w_keepalive).name)
+ if self.w_keepalive is not None:
+ name = self.space.type(self.w_keepalive).name
+ else:
+ name = "(released)"
+ return "buffer len %d from '%s' object" % (self.length, name)
def enter_exit(self, exit_now):
- pass # for now, no effect on PyPy
+ # for now, limited effect on PyPy
+ if exit_now:
+ self.w_keepalive = None
class W_CDataGCP(W_CData):
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -25,7 +25,7 @@
assert isinstance(ctptr, W_CTypePointer)
W_CTypePtrOrArray.__init__(self, space, arraysize, extra, 0,
ctptr.ctitem)
- self.length = length
+ self.length = length # -1 if no length is given, e.g. 'int[]'
self.ctptr = ctptr
def _alignof(self):
@@ -86,7 +86,7 @@
def _check_subscript_index(self, w_cdata, i):
space = self.space
if i < 0:
- raise oefmt(space.w_IndexError, "negative index not supported")
+ raise oefmt(space.w_IndexError, "negative index")
if i >= w_cdata.get_array_length():
raise oefmt(space.w_IndexError,
"index too large for cdata '%s' (expected %d < %d)",
@@ -96,7 +96,7 @@
def _check_slice_index(self, w_cdata, start, stop):
space = self.space
if start < 0:
- raise oefmt(space.w_IndexError, "negative index not supported")
+ raise oefmt(space.w_IndexError, "negative index")
if stop > w_cdata.get_array_length():
raise oefmt(space.w_IndexError,
"index too large (expected %d <= %d)",
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -328,7 +328,8 @@
@unwrap_spec(require_writable=int)
- def descr_from_buffer(self, w_python_buffer, require_writable=0):
+ def descr_from_buffer(self, w_cdecl, w_python_buffer=None,
+ require_writable=0):
"""\
Return a <cdata 'char[]'> that points to the data of the given Python
object, which must support the buffer interface. Note that this is
@@ -337,9 +338,13 @@
containing large quantities of raw data in some other format, like
'array.array' or numpy arrays."""
#
- w_ctchara = newtype._new_chara_type(self.space)
- return func._from_buffer(self.space, w_ctchara, w_python_buffer,
- require_writable)
+ if w_python_buffer is None:
+ w_python_buffer = w_cdecl
+ w_ctype = newtype._new_chara_type(self.space)
+ else:
+ w_ctype = self.ffi_type(w_cdecl, ACCEPT_STRING | ACCEPT_CTYPE)
+ return func.from_buffer(self.space, w_ctype, w_python_buffer,
+ require_writable)
@unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -112,16 +112,10 @@
@unwrap_spec(w_ctype=ctypeobj.W_CType, require_writable=int)
def from_buffer(space, w_ctype, w_x, require_writable=0):
- from pypy.module._cffi_backend import ctypearray, ctypeprim
- #
- if (not isinstance(w_ctype, ctypearray.W_CTypeArray) or
- not isinstance(w_ctype.ctptr.ctitem, ctypeprim.W_CTypePrimitiveChar)):
- raise oefmt(space.w_TypeError,
- "needs 'char[]', got '%s'", w_ctype.name)
- #
- return _from_buffer(space, w_ctype, w_x, require_writable)
-
-def _from_buffer(space, w_ctype, w_x, require_writable):
+ from pypy.module._cffi_backend import ctypearray
+ if not isinstance(w_ctype, ctypearray.W_CTypeArray):
+ raise oefmt(space.w_TypeError, "expected an array ctype, got '%s'",
+ w_ctype.name)
if space.isinstance_w(w_x, space.w_unicode):
raise oefmt(space.w_TypeError,
"from_buffer() cannot return the address of a unicode object")
@@ -140,7 +134,37 @@
"buffer interface but cannot be rendered as a plain "
"raw address on PyPy", w_x)
#
- return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x)
+ buffersize = buf.getlength()
+ arraylength = w_ctype.length
+ if arraylength >= 0:
+ # it's an array with a fixed length; make sure that the
+ # buffer contains enough bytes.
+ if buffersize < w_ctype.size:
+ raise oefmt(space.w_ValueError,
+ "buffer is too small (%d bytes) for '%s' (%d bytes)",
+ buffersize, w_ctype.name, w_ctype.size)
+ else:
+ # it's an open 'array[]'
+ itemsize = w_ctype.ctitem.size
+ if itemsize == 1:
+ # fast path, performance only
+ arraylength = buffersize
+ elif itemsize > 0:
+ # give it as many items as fit the buffer. Ignore a
+ # partial last element.
+ arraylength = buffersize / itemsize
+ else:
+ # it's an array 'empty[]'. Unsupported obscure case:
+ # the problem is that setting the length of the result
+ # to anything large (like SSIZE_T_MAX) is dangerous,
+ # because if someone tries to loop over it, it will
+ # turn effectively into an infinite loop.
+ raise oefmt(space.w_ZeroDivisionError,
+ "from_buffer('%s', ..): the actual length of the array "
+ "cannot be computed", w_ctype.name)
+ #
+ return cdataobj.W_CDataFromBuffer(space, _cdata, arraylength,
+ w_ctype, buf, w_x)
# ____________________________________________________________
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3742,6 +3742,64 @@
p1[0] = b"g"
assert ba == b"goo"
+def test_from_buffer_types():
+ BInt = new_primitive_type("int")
+ BIntP = new_pointer_type(BInt)
+ BIntA = new_array_type(BIntP, None)
+ lst = [-12345678, 87654321, 489148]
+ bytestring = buffer(newp(BIntA, lst))[:] + b'XYZ'
+ #
+ p1 = from_buffer(BIntA, bytestring) # int[]
+ assert typeof(p1) is BIntA
+ assert len(p1) == 3
+ assert p1[0] == lst[0]
+ assert p1[1] == lst[1]
+ assert p1[2] == lst[2]
+ py.test.raises(IndexError, "p1[3]")
+ py.test.raises(IndexError, "p1[-1]")
+ #
+ py.test.raises(TypeError, from_buffer, BInt, bytestring)
+ py.test.raises(TypeError, from_buffer, BIntP, bytestring)
+ #
+ BIntA2 = new_array_type(BIntP, 2)
+ p2 = from_buffer(BIntA2, bytestring) # int[2]
+ assert typeof(p2) is BIntA2
+ assert len(p2) == 2
+ assert p2[0] == lst[0]
+ assert p2[1] == lst[1]
+ py.test.raises(IndexError, "p2[2]")
+ py.test.raises(IndexError, "p2[-1]")
+ assert p2 == p1
+ #
+ BIntA4 = new_array_type(BIntP, 4) # int[4]: too big
+ py.test.raises(ValueError, from_buffer, BIntA4, bytestring)
+ #
+ BStruct = new_struct_type("foo")
+ complete_struct_or_union(BStruct, [('a1', BInt, -1),
+ ('a2', BInt, -1)])
+ BStructP = new_pointer_type(BStruct)
+ BStructA = new_array_type(BStructP, None)
+ p1 = from_buffer(BStructA, bytestring) # struct[]
+ assert len(p1) == 1
+ assert typeof(p1) is BStructA
+ assert p1[0].a1 == lst[0]
+ assert p1[0].a2 == lst[1]
+ py.test.raises(IndexError, "p1[1]")
+ #
+ BEmptyStruct = new_struct_type("empty")
+ complete_struct_or_union(BEmptyStruct, [], Ellipsis, 0)
+ assert sizeof(BEmptyStruct) == 0
+ BEmptyStructP = new_pointer_type(BEmptyStruct)
+ BEmptyStructA = new_array_type(BEmptyStructP, None)
+ py.test.raises(ZeroDivisionError, from_buffer, # empty[]
+ BEmptyStructA, bytestring)
+ #
+ BEmptyStructA5 = new_array_type(BEmptyStructP, 5)
+ p1 = from_buffer(BEmptyStructA5, bytestring) # struct empty[5]
+ assert typeof(p1) is BEmptyStructA5
+ assert len(p1) == 5
+ assert cast(BIntP, p1) == from_buffer(BIntA, bytestring)
+
def test_memmove():
Short = new_primitive_type("short")
ShortA = new_array_type(new_pointer_type(Short), None)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -282,19 +282,31 @@
import _cffi_backend as _cffi1_backend
import array
ffi = _cffi1_backend.FFI()
- a = array.array('H', [10000, 20000, 30000])
+ a = array.array('H', [10000, 20000, 30000, 40000])
c = ffi.from_buffer(a)
assert ffi.typeof(c) is ffi.typeof("char[]")
+ assert len(c) == 8
ffi.cast("unsigned short *", c)[1] += 500
- assert list(a) == [10000, 20500, 30000]
- assert c == ffi.from_buffer(a, True)
+ assert list(a) == [10000, 20500, 30000, 40000]
+ raises(TypeError, ffi.from_buffer, a, True)
+ assert c == ffi.from_buffer("char[]", a, True)
assert c == ffi.from_buffer(a, require_writable=True)
#
+ c = ffi.from_buffer("unsigned short[]", a)
+ assert len(c) == 4
+ assert c[1] == 20500
+ #
+ c = ffi.from_buffer("unsigned short[2][2]", a)
+ assert len(c) == 2
+ assert len(c[0]) == 2
+ assert c[0][1] == 20500
+ #
p = ffi.from_buffer(b"abcd")
assert p[2] == b"c"
#
- assert p == ffi.from_buffer(b"abcd", False)
- raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True)
+ assert p == ffi.from_buffer(b"abcd", require_writable=False)
+ raises((TypeError, BufferError), ffi.from_buffer,
+ "char[]", b"abcd", True)
raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
require_writable=True)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -20,7 +20,7 @@
assert log.result % 1000 == 0
loop, = log.loops_by_filename(self.filepath)
ops = loop.ops_by_id('look')
- assert log.opnames(ops) == []
+ assert log.opnames(ops) == ['guard_nonnull_class']
def test_identitydict(self):
def fn(n):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -407,6 +407,7 @@
i138 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=...)
check_memory_error(i138)
setfield_gc(p132, i138, descr=...)
+ setfield_gc(p132, 0, descr=...)
setfield_gc(p132, ConstPtr(ptr139), descr=...)
setfield_gc(p132, -1, descr=...)
setfield_gc(p0, p133, descr=...)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -244,6 +244,8 @@
f80 = raw_load_f(i67, i79, descr=<ArrayF 8>)
i81 = int_add(i71, 1)
--TICK--
+ i92 = int_le(i33, _)
+ guard_true(i92, descr=...)
jump(..., descr=...)
""")
@@ -283,6 +285,8 @@
f86 = float_add(f74, f85)
i87 = int_add(i76, 1)
--TICK--
+ i98 = int_le(i36, _)
+ guard_true(i98, descr=...)
jump(..., descr=...)
""")
@@ -390,6 +394,8 @@
assert log.result == [0.] * N
loop, = log.loops_by_filename(self.filepath)
assert loop.match("""
+ i4 = int_lt(i91, 0)
+ guard_false(i4, descr=...)
i92 = int_ge(i91, i37)
guard_false(i92, descr=...)
i93 = int_add(i91, 1)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -113,6 +113,7 @@
i12 = int_is_true(i4)
guard_true(i12, descr=...)
guard_not_invalidated(descr=...)
+ guard_nonnull_class(p10, ConstClass(W_IntObject), descr=...)
i10p = getfield_gc_i(p10, descr=...)
i10 = int_mul_ovf(2, i10p)
guard_no_overflow(descr=...)
@@ -146,12 +147,16 @@
RANGE_ITER_STEP_1 = """
guard_not_invalidated?
# W_IntRangeStepOneIterator.next()
+ i80 = int_lt(i11, 0)
+ guard_false(i80, descr=...)
i16 = int_lt(i11, i12)
guard_true(i16, descr=...)
i20 = int_add(i11, 1)
setfield_gc(p4, i20, descr=<.* .*W_IntRangeIterator.inst_current .*>)
guard_not_invalidated?
i21 = force_token()
+ i89 = int_lt(0, i9)
+ guard_true(i89, descr=...)
i88 = int_sub(i9, 1)
# Compared with pypy2, we get these two operations extra.
@@ -186,6 +191,8 @@
assert log.result == 1000 * 999 / 2
loop, = log.loops_by_filename(self.filepath)
assert loop.match(self.RANGE_ITER_STEP_1)
+ i94 = int_lt(0, i9)
+ guard_true(i94, descr=...)
def test_chain_of_guards(self):
src = """
More information about the pypy-commit
mailing list