[pypy-commit] pypy unicode-utf8: merge default into branch
mattip
pypy.commits at gmail.com
Sun Jan 6 02:57:13 EST 2019
Author: Matti Picus <matti.picus at gmail.com>
Branch: unicode-utf8
Changeset: r95583:a05d741a4652
Date: 2019-01-06 09:33 +0200
http://bitbucket.org/pypy/pypy/changeset/a05d741a4652/
Log: merge default into branch
diff --git a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
--- a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
+++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
@@ -338,6 +338,13 @@
py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
require_writable=True)
+ def test_release(self):
+ ffi = FFI()
+ p = ffi.new("int[]", 123)
+ ffi.release(p)
+ # here, reading p[0] might give garbage or segfault...
+ ffi.release(p) # no effect
+
def test_memmove(self):
ffi = FFI()
p = ffi.new("short[]", [-1234, -2345, -3456, -4567, -5678])
diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
--- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1457,6 +1457,35 @@
import gc; gc.collect(); gc.collect(); gc.collect()
assert seen == [3]
+ def test_release(self):
+ p = ffi.new("int[]", 123)
+ ffi.release(p)
+ # here, reading p[0] might give garbage or segfault...
+ ffi.release(p) # no effect
+
+ def test_release_new_allocator(self):
+ seen = []
+ def myalloc(size):
+ seen.append(size)
+ return ffi.new("char[]", b"X" * size)
+ def myfree(raw):
+ seen.append(raw)
+ alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree)
+ p = alloc2("int[]", 15)
+ assert seen == [15 * 4]
+ ffi.release(p)
+ assert seen == [15 * 4, p]
+ ffi.release(p) # no effect
+ assert seen == [15 * 4, p]
+ #
+ del seen[:]
+ p = alloc2("struct ab *")
+ assert seen == [2 * 4]
+ ffi.release(p)
+ assert seen == [2 * 4, p]
+ ffi.release(p) # no effect
+ assert seen == [2 * 4, p]
+
def test_CData_CType(self):
assert isinstance(ffi.cast("int", 0), ffi.CData)
assert isinstance(ffi.new("int *"), ffi.CData)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -530,6 +530,9 @@
def from_handle(self, x):
return self._backend.from_handle(x)
+ def release(self, x):
+ self._backend.release(x)
+
def set_unicode(self, enabled_flag):
"""Windows: if 'enabled_flag' is True, enable the UNICODE and
_UNICODE defines in C, and declare the types like TCHAR and LPTCSTR
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,6 +16,13 @@
except ImportError:
lock = None
+def _workaround_for_static_import_finders():
+ # Issue #392: packaging tools like cx_Freeze can not find these
+ # because pycparser uses exec dynamic import. This is an obscure
+ # workaround. This function is never called.
+ import pycparser.yacctab
+ import pycparser.lextab
+
CDEF_SOURCE_STRING = "<cdef source string>"
_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
re.DOTALL | re.MULTILINE)
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -48,3 +48,8 @@
assert len(b) == 16
assert s == "abc123you and me"
assert b.build() == s
+
+ def test_encode(self):
+ from __pypy__.builders import UnicodeBuilder
+ b = UnicodeBuilder()
+ raises(UnicodeDecodeError, b.append, b'\xc0')
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -52,6 +52,7 @@
'unpack': 'func.unpack',
'buffer': 'cbuffer.MiniBuffer',
'memmove': 'func.memmove',
+ 'release': 'func.release',
'get_errno': 'cerrno.get_errno',
'set_errno': 'cerrno.set_errno',
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -483,6 +483,18 @@
def get_structobj(self):
return None
+ def enter_exit(self, exit_now):
+ raise oefmt(self.space.w_ValueError,
+ "only 'cdata' object from ffi.new(), ffi.gc() or ffi.from_buffer() "
+ "can be used with the 'with' keyword or ffi.release()")
+
+ def descr_enter(self):
+ self.enter_exit(False)
+ return self
+
+ def descr_exit(self, args_w):
+ self.enter_exit(True)
+
class W_CDataMem(W_CData):
"""This is used only by the results of cffi.cast('int', x)
@@ -535,14 +547,33 @@
def get_structobj(self):
return self
+ def enter_exit(self, exit_now):
+ from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+ if not isinstance(self.ctype, W_CTypePtrOrArray):
+ W_CData.enter_exit(self, exit_now)
+ elif exit_now:
+ self._do_exit()
+
+ def _do_exit(self):
+ raise NotImplementedError
+
class W_CDataNewStd(W_CDataNewOwning):
"""Subclass using the standard allocator, lltype.malloc()/lltype.free()"""
- _attrs_ = []
+ _attrs_ = ['explicitly_freed']
+ explicitly_freed = False
@rgc.must_be_light_finalizer
def __del__(self):
- lltype.free(self._ptr, flavor='raw')
+ if not self.explicitly_freed:
+ lltype.free(self._ptr, flavor='raw')
+
+ def _do_exit(self):
+ if not self.explicitly_freed:
+ rgc.add_memory_pressure(-self._sizeof(), self)
+ self.explicitly_freed = True
+ rgc.may_ignore_finalizer(self)
+ lltype.free(self._ptr, flavor='raw')
class W_CDataNewNonStd(W_CDataNewOwning):
@@ -550,7 +581,16 @@
_attrs_ = ['w_raw_cdata', 'w_free']
def _finalize_(self):
- self.space.call_function(self.w_free, self.w_raw_cdata)
+ if self.w_free is not None:
+ self.space.call_function(self.w_free, self.w_raw_cdata)
+
+ def _do_exit(self):
+ w_free = self.w_free
+ if w_free is not None:
+ rgc.add_memory_pressure(-self._sizeof(), self)
+ self.w_free = None
+ self.may_unregister_rpython_finalizer(self.space)
+ self.space.call_function(w_free, self.w_raw_cdata)
class W_CDataPtrToStructOrUnion(W_CData):
@@ -580,6 +620,12 @@
else:
return None
+ def enter_exit(self, exit_now):
+ if exit_now:
+ structobj = self.structobj
+ if isinstance(structobj, W_CDataNewOwning):
+ structobj._do_exit()
+
class W_CDataSliced(W_CData):
"""Subclass with an explicit length, for slices."""
@@ -634,6 +680,9 @@
return "buffer len %d from '%s' object" % (
self.length, self.space.type(self.w_keepalive).name)
+ def enter_exit(self, exit_now):
+ pass # for now, no effect on PyPy
+
class W_CDataGCP(W_CData):
"""For ffi.gc()."""
@@ -647,6 +696,9 @@
self.register_finalizer(space)
def _finalize_(self):
+ self.invoke_finalizer()
+
+ def invoke_finalizer(self):
w_destructor = self.w_destructor
if w_destructor is not None:
self.w_destructor = None
@@ -656,6 +708,11 @@
self.w_destructor = None
self.may_unregister_rpython_finalizer(self.space)
+ def enter_exit(self, exit_now):
+ if exit_now:
+ self.may_unregister_rpython_finalizer(self.space)
+ self.invoke_finalizer()
+
W_CData.typedef = TypeDef(
'_cffi_backend.CData',
@@ -686,5 +743,7 @@
__iter__ = interp2app(W_CData.iter),
__weakref__ = make_weakref_descr(W_CData),
__dir__ = interp2app(W_CData.dir),
+ __enter__ = interp2app(W_CData.descr_enter),
+ __exit__ = interp2app(W_CData.descr_exit),
)
W_CData.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -703,6 +703,16 @@
pass
return w_res
+ @unwrap_spec(w_cdata=W_CData)
+ def descr_release(self, w_cdata):
+ """\
+Release now the resources held by a 'cdata' object from ffi.new(),
+ffi.gc() or ffi.from_buffer(). The cdata object must not be used
+afterwards.
+
+'ffi.release(cdata)' is equivalent to 'cdata.__exit__()'."""
+ w_cdata.enter_exit(True)
+
class W_InitOnceLock(W_Root):
def __init__(self, space):
@@ -777,6 +787,7 @@
new_allocator = interp2app(W_FFIObject.descr_new_allocator),
new_handle = interp2app(W_FFIObject.descr_new_handle),
offsetof = interp2app(W_FFIObject.descr_offsetof),
+ release = interp2app(W_FFIObject.descr_release),
sizeof = interp2app(W_FFIObject.descr_sizeof),
string = interp2app(W_FFIObject.descr_string),
typeof = interp2app(W_FFIObject.descr_typeof),
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -264,3 +264,7 @@
@unwrap_spec(w_cdata=cdataobj.W_CData, size=int)
def gcp(space, w_cdata, w_destructor, size=0):
return w_cdata.with_gc(w_destructor, size)
+
+ at unwrap_spec(w_cdata=cdataobj.W_CData)
+def release(space, w_cdata):
+ w_cdata.enter_exit(True)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -4074,3 +4074,114 @@
assert_eq(cast(t5, 7.0), cast(t3, 7))
assert_lt(cast(t5, 3.1), 3.101)
assert_gt(cast(t5, 3.1), 3)
+
+def test_explicit_release_new():
+ # release() on a ffi.new() object has no effect on CPython, but
+ # really releases memory on PyPy. We can't test that effect
+ # though, because a released cdata is not marked.
+ BIntP = new_pointer_type(new_primitive_type("int"))
+ p = newp(BIntP)
+ p[0] = 42
+ py.test.raises(IndexError, "p[1]")
+ release(p)
+ # here, reading p[0] might give garbage or segfault...
+ release(p) # no effect
+ #
+ BStruct = new_struct_type("struct foo")
+ BStructP = new_pointer_type(BStruct)
+ complete_struct_or_union(BStruct, [('p', BIntP, -1)])
+ pstruct = newp(BStructP)
+ assert pstruct.p == cast(BIntP, 0)
+ release(pstruct)
+ # here, reading pstruct.p might give garbage or segfault...
+ release(pstruct) # no effect
+
+def test_explicit_release_new_contextmgr():
+ BIntP = new_pointer_type(new_primitive_type("int"))
+ with newp(BIntP) as p:
+ p[0] = 42
+ assert p[0] == 42
+ # here, reading p[0] might give garbage or segfault...
+ release(p) # no effect
+
+def test_explicit_release_badtype():
+ BIntP = new_pointer_type(new_primitive_type("int"))
+ p = cast(BIntP, 12345)
+ py.test.raises(ValueError, release, p)
+ py.test.raises(ValueError, release, p)
+ BStruct = new_struct_type("struct foo")
+ BStructP = new_pointer_type(BStruct)
+ complete_struct_or_union(BStruct, [('p', BIntP, -1)])
+ pstruct = newp(BStructP)
+ py.test.raises(ValueError, release, pstruct[0])
+
+def test_explicit_release_badtype_contextmgr():
+ BIntP = new_pointer_type(new_primitive_type("int"))
+ p = cast(BIntP, 12345)
+ py.test.raises(ValueError, "with p: pass")
+ py.test.raises(ValueError, "with p: pass")
+
+def test_explicit_release_gc():
+ BIntP = new_pointer_type(new_primitive_type("int"))
+ seen = []
+ intp1 = newp(BIntP, 12345)
+ p1 = cast(BIntP, intp1)
+ p = gcp(p1, seen.append)
+ assert seen == []
+ release(p)
+ assert seen == [p1]
+ assert p1[0] == 12345
+ assert p[0] == 12345 # true so far, but might change to raise RuntimeError
+ release(p) # no effect
+
+def test_explicit_release_gc_contextmgr():
+ BIntP = new_pointer_type(new_primitive_type("int"))
+ seen = []
+ intp1 = newp(BIntP, 12345)
+ p1 = cast(BIntP, intp1)
+ p = gcp(p1, seen.append)
+ with p:
+ assert p[0] == 12345
+ assert seen == []
+ assert seen == [p1]
+ assert p1[0] == 12345
+ assert p[0] == 12345 # true so far, but might change to raise RuntimeError
+ release(p) # no effect
+
+def test_explicit_release_from_buffer():
+ a = bytearray(b"xyz")
+ BChar = new_primitive_type("char")
+ BCharP = new_pointer_type(BChar)
+ BCharA = new_array_type(BCharP, None)
+ p = from_buffer(BCharA, a)
+ assert p[2] == b"z"
+ release(p)
+ assert p[2] == b"z" # true so far, but might change to raise RuntimeError
+ release(p) # no effect
+
+def test_explicit_release_from_buffer_contextmgr():
+ a = bytearray(b"xyz")
+ BChar = new_primitive_type("char")
+ BCharP = new_pointer_type(BChar)
+ BCharA = new_array_type(BCharP, None)
+ p = from_buffer(BCharA, a)
+ with p:
+ assert p[2] == b"z"
+ assert p[2] == b"z" # true so far, but might change to raise RuntimeError
+ release(p) # no effect
+
+def test_explicit_release_bytearray_on_cpython():
+ if '__pypy__' in sys.builtin_module_names:
+ py.test.skip("pypy's bytearray are never locked")
+ a = bytearray(b"xyz")
+ BChar = new_primitive_type("char")
+ BCharP = new_pointer_type(BChar)
+ BCharA = new_array_type(BCharP, None)
+ a += b't' * 10
+ p = from_buffer(BCharA, a)
+ py.test.raises(BufferError, "a += b'u' * 100")
+ release(p)
+ a += b'v' * 100
+ release(p) # no effect
+ a += b'w' * 1000
+ assert a == bytearray(b"xyz" + b't' * 10 + b'v' * 100 + b'w' * 1000)
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -2107,3 +2107,36 @@
else:
assert lib.__loader__ is None
assert lib.__spec__ is None
+
+ def test_release(self):
+ ffi, lib = self.prepare("", "test_release", "")
+ p = ffi.new("int[]", 123)
+ ffi.release(p)
+ # here, reading p[0] might give garbage or segfault...
+ ffi.release(p) # no effect
+
+ def test_release_new_allocator(self):
+ ffi, lib = self.prepare("struct ab { int a, b; };",
+ "test_release_new_allocator",
+ "struct ab { int a, b; };")
+ seen = []
+ def myalloc(size):
+ seen.append(size)
+ return ffi.new("char[]", b"X" * size)
+ def myfree(raw):
+ seen.append(raw)
+ alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree)
+ p = alloc2("int[]", 15)
+ assert seen == [15 * 4]
+ ffi.release(p)
+ assert seen == [15 * 4, p]
+ ffi.release(p) # no effect
+ assert seen == [15 * 4, p]
+ #
+ del seen[:]
+ p = alloc2("struct ab *")
+ assert seen == [2 * 4]
+ ffi.release(p)
+ assert seen == [2 * 4, p]
+ ffi.release(p) # no effect
+ assert seen == [2 * 4, p]
More information about the pypy-commit
mailing list