[pypy-commit] pypy py3.5: hg merge
arigo
pypy.commits at gmail.com
Sun Feb 5 16:46:22 EST 2017
Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r89963:748aa3022295
Date: 2017-02-05 22:45 +0100
http://bitbucket.org/pypy/pypy/changeset/748aa3022295/
Log: hg merge
diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -31,6 +31,26 @@
"WeakKeyDictionary", "ReferenceError", "ReferenceType", "ProxyType",
"CallableProxyType", "ProxyTypes", "WeakValueDictionary", 'WeakSet']
+try:
+ from __pypy__ import delitem_if_value_is as _delitem_if_value_is
+except ImportError:
+ def _delitem_if_value_is(d, key, value):
+ try:
+ if self.data[key] is value: # fall-back: there is a potential
+ # race condition in multithreaded programs HERE
+ del self.data[key]
+ except KeyError:
+ pass
+
+def _remove_dead_weakref(d, key):
+ try:
+ wr = d[key]
+ except KeyError:
+ pass
+ else:
+ if wr() is None:
+ _delitem_if_value_is(d, key, wr)
+
class WeakValueDictionary(UserDict.UserDict):
"""Mapping class that references values weakly.
@@ -58,14 +78,9 @@
if self._iterating:
self._pending_removals.append(wr.key)
else:
- # Changed this for PyPy: made more resistent. The
- # issue is that in some corner cases, self.data
- # might already be changed or removed by the time
- # this weakref's callback is called. If that is
- # the case, we don't want to randomly kill an
- # unrelated entry.
- if self.data.get(wr.key) is wr:
- del self.data[wr.key]
+ # Atomic removal is necessary since this function
+ # can be called asynchronously by the GC
+ _delitem_if_value_is(self.data, wr.key, wr)
self._remove = remove
# A list of keys to be removed
self._pending_removals = []
@@ -78,7 +93,8 @@
# We shouldn't encounter any KeyError, because this method should
# always be called *before* mutating the dict.
while l:
- del d[l.pop()]
+ key = l.pop()
+ _remove_dead_weakref(d, key)
def __getitem__(self, key):
o = self.data[key]()
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -78,6 +78,7 @@
'add_memory_pressure' : 'interp_magic.add_memory_pressure',
'newdict' : 'interp_dict.newdict',
'reversed_dict' : 'interp_dict.reversed_dict',
+ 'delitem_if_value_is' : 'interp_dict.delitem_if_value_is',
'move_to_end' : 'interp_dict.move_to_end',
'strategy' : 'interp_magic.strategy', # dict,set,list
'set_debug' : 'interp_magic.set_debug',
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -59,3 +59,16 @@
if not isinstance(w_obj, W_DictMultiObject):
raise OperationError(space.w_TypeError, space.w_None)
return w_obj.nondescr_move_to_end(space, w_key, last)
+
+def delitem_if_value_is(space, w_obj, w_key, w_value):
+ """Atomic equivalent to: 'if dict.get(key) is value: del dict[key]'.
+
+ SPECIAL USE CASES ONLY! Avoid using on dicts which are specialized,
+ e.g. to int or str keys, because it switches to the object strategy.
+ Also, the 'is' operation is really pointer equality, so avoid using
+ it if 'value' is an immutable object like int or str.
+ """
+ from pypy.objspace.std.dictmultiobject import W_DictMultiObject
+ if not isinstance(w_obj, W_DictMultiObject):
+ raise OperationError(space.w_TypeError, space.w_None)
+ return w_obj.nondescr_delitem_if_value_is(space, w_key, w_value)
diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -40,6 +40,7 @@
import pypy.module.cpyext.pyerrors
import pypy.module.cpyext.typeobject
import pypy.module.cpyext.object
+import pypy.module.cpyext.buffer
import pypy.module.cpyext.bytesobject
import pypy.module.cpyext.bytearrayobject
import pypy.module.cpyext.tupleobject
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -126,7 +126,7 @@
Py_TPFLAGS_HEAPTYPE
Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_MAX_NDIMS
Py_CLEANUP_SUPPORTED
-PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES
+PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE PyBUF_SIMPLE
""".split()
for name in constant_names:
setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/buffer.py
@@ -0,0 +1,72 @@
+from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.interpreter.error import oefmt
+from pypy.module.cpyext.api import (
+ cpython_api, Py_buffer, Py_ssize_t, Py_ssize_tP, CONST_STRINGP,
+ generic_cpy_call,
+ PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES, PyBUF_SIMPLE)
+from pypy.module.cpyext.pyobject import PyObject, Py_IncRef, Py_DecRef
+
+ at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
+def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
+ """Returns a pointer to a read-only memory location usable as
+ character-based input. The obj argument must support the single-segment
+ character buffer interface. On success, returns 0, sets buffer to the
+ memory location and size to the buffer length. Returns -1 and sets a
+ TypeError on error.
+ """
+ pto = obj.c_ob_type
+ pb = pto.c_tp_as_buffer
+ if not (pb and pb.c_bf_getbuffer):
+ raise oefmt(space.w_TypeError,
+ "expected an object with the buffer interface")
+ with lltype.scoped_alloc(Py_buffer) as view:
+ ret = generic_cpy_call(
+ space, pb.c_bf_getbuffer,
+ obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
+ if rffi.cast(lltype.Signed, ret) == -1:
+ return -1
+
+ bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
+ sizep[0] = view.c_len
+
+ if pb.c_bf_releasebuffer:
+ generic_cpy_call(space, pb.c_bf_releasebuffer,
+ obj, view)
+ Py_DecRef(space, view.c_obj)
+ return 0
+
+
+ at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
+ lltype.Signed, lltype.Signed], rffi.INT, error=-1)
+def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
+ """
+ Fills in a buffer-info structure correctly for an exporter that can only
+ share a contiguous chunk of memory of "unsigned bytes" of the given
+ length. Returns 0 on success and -1 (with raising an error) on error.
+ """
+ flags = rffi.cast(lltype.Signed, flags)
+ if flags & PyBUF_WRITABLE and readonly:
+ raise oefmt(space.w_ValueError, "Object is not writable")
+ view.c_buf = buf
+ view.c_len = length
+ view.c_obj = obj
+ if obj:
+ Py_IncRef(space, obj)
+ view.c_itemsize = 1
+ rffi.setintfield(view, 'c_readonly', readonly)
+ rffi.setintfield(view, 'c_ndim', 1)
+ view.c_format = lltype.nullptr(rffi.CCHARP.TO)
+ if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
+ view.c_format = rffi.str2charp("B")
+ view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
+ if (flags & PyBUF_ND) == PyBUF_ND:
+ view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
+ view.c_shape[0] = view.c_len
+ view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
+ if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
+ view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
+ view.c_strides[0] = view.c_itemsize
+ view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
+ view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
+
+ return 0
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -1,13 +1,11 @@
from rpython.rtyper.lltypesystem import rffi, lltype
from pypy.module.cpyext.api import (
- cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP,
- PyVarObject, Py_buffer, size_t, slot_function, cts,
- PyBUF_FORMAT, PyBUF_ND, PyBUF_STRIDES,
+ cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t,
+ PyVarObject, size_t, slot_function, cts,
Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT,
- Py_GE, CONST_STRING, CONST_STRINGP, FILEP, fwrite)
+ Py_GE, CONST_STRING, FILEP, fwrite)
from pypy.module.cpyext.pyobject import (
- PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef,
- get_typedescr, _Py_NewReference)
+ PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef, get_typedescr)
from pypy.module.cpyext.typeobject import PyTypeObjectPtr
from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
from pypy.objspace.std.typeobject import W_TypeObject
@@ -16,10 +14,6 @@
import pypy.module.__builtin__.operation as operation
-# from include/object.h
-PyBUF_SIMPLE = 0x0000
-PyBUF_WRITABLE = 0x0001
-
@cpython_api([size_t], rffi.VOIDP)
def PyObject_Malloc(space, size):
# returns non-zero-initialized memory, like CPython
@@ -444,36 +438,6 @@
is active then NULL is returned but PyErr_Occurred() will return false."""
return space.call_function(space.builtin.get('dir'), w_o)
- at cpython_api([PyObject, CONST_STRINGP, Py_ssize_tP], rffi.INT_real, error=-1)
-def PyObject_AsCharBuffer(space, obj, bufferp, sizep):
- """Returns a pointer to a read-only memory location usable as
- character-based input. The obj argument must support the single-segment
- character buffer interface. On success, returns 0, sets buffer to the
- memory location and size to the buffer length. Returns -1 and sets a
- TypeError on error.
- """
- pto = obj.c_ob_type
-
- pb = pto.c_tp_as_buffer
- if not (pb and pb.c_bf_getbuffer):
- raise oefmt(space.w_TypeError,
- "expected an object with the buffer interface")
- with lltype.scoped_alloc(Py_buffer) as view:
- ret = generic_cpy_call(
- space, pb.c_bf_getbuffer,
- obj, view, rffi.cast(rffi.INT_real, PyBUF_SIMPLE))
- if rffi.cast(lltype.Signed, ret) == -1:
- return -1
-
- bufferp[0] = rffi.cast(rffi.CCHARP, view.c_buf)
- sizep[0] = view.c_len
-
- if pb.c_bf_releasebuffer:
- generic_cpy_call(space, pb.c_bf_releasebuffer,
- obj, view)
- Py_DecRef(space, view.c_obj)
- return 0
-
# Also in include/object.h
Py_PRINT_RAW = 1 # No string quotes etc.
@@ -493,41 +457,3 @@
with rffi.scoped_nonmovingbuffer(data) as buf:
fwrite(buf, 1, count, fp)
return 0
-
-
-PyBUF_WRITABLE = 0x0001 # Copied from object.h
-
- at cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t,
- lltype.Signed, lltype.Signed], rffi.INT, error=-1)
-def PyBuffer_FillInfo(space, view, obj, buf, length, readonly, flags):
- """
- Fills in a buffer-info structure correctly for an exporter that can only
- share a contiguous chunk of memory of "unsigned bytes" of the given
- length. Returns 0 on success and -1 (with raising an error) on error.
- """
- flags = rffi.cast(lltype.Signed, flags)
- if flags & PyBUF_WRITABLE and readonly:
- raise oefmt(space.w_ValueError, "Object is not writable")
- view.c_buf = buf
- view.c_len = length
- view.c_obj = obj
- if obj:
- Py_IncRef(space, obj)
- view.c_itemsize = 1
- rffi.setintfield(view, 'c_readonly', readonly)
- rffi.setintfield(view, 'c_ndim', 1)
- view.c_format = lltype.nullptr(rffi.CCHARP.TO)
- if (flags & PyBUF_FORMAT) == PyBUF_FORMAT:
- view.c_format = rffi.str2charp("B")
- view.c_shape = lltype.nullptr(Py_ssize_tP.TO)
- if (flags & PyBUF_ND) == PyBUF_ND:
- view.c_shape = rffi.cast(Py_ssize_tP, view.c__shape)
- view.c_shape[0] = view.c_len
- view.c_strides = lltype.nullptr(Py_ssize_tP.TO)
- if (flags & PyBUF_STRIDES) == PyBUF_STRIDES:
- view.c_strides = rffi.cast(Py_ssize_tP, view.c__strides)
- view.c_strides[0] = view.c_itemsize
- view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO)
- view.c_internal = lltype.nullptr(rffi.VOIDP.TO)
-
- return 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -548,14 +548,14 @@
@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
def bytes_getbuffer(space, w_str, view, flags):
from pypy.module.cpyext.bytesobject import PyBytes_AsString
- from pypy.module.cpyext.object import PyBuffer_FillInfo
+ from pypy.module.cpyext.buffer import PyBuffer_FillInfo
c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str))
return PyBuffer_FillInfo(space, view, w_str, c_buf,
space.len_w(w_str), 1, flags)
@slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1)
def bf_getbuffer(space, w_obj, view, flags):
- from pypy.module.cpyext.object import PyBuffer_FillInfo
+ from pypy.module.cpyext.buffer import PyBuffer_FillInfo
buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags))
c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
return PyBuffer_FillInfo(space, view, w_obj, c_buf,
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -263,6 +263,14 @@
for i in range(len(keys_w)):
self.setitem(keys_w[i], values_w[i])
+ def nondescr_delitem_if_value_is(self, space, w_key, w_value):
+ """Not exposed directly to app-level, but used by
+ _weakref._remove_dead_weakref and via __pypy__.delitem_if_value_is().
+ """
+ strategy = self.ensure_object_strategy()
+ d = strategy.unerase(self.dstorage)
+ objectmodel.delitem_if_value_is(d, w_key, w_value)
+
def descr_clear(self, space):
"""D.clear() -> None. Remove all items from D."""
self.clear()
@@ -314,11 +322,12 @@
F: D[k] = F[k]"""
init_or_update(space, self, __args__, 'dict.update')
- def ensure_object_strategy(self): # for cpyext
+ def ensure_object_strategy(self): # also called by cpyext
object_strategy = self.space.fromcache(ObjectDictStrategy)
strategy = self.get_strategy()
if strategy is not object_strategy:
strategy.switch_to_object_strategy(self)
+ return object_strategy
class W_DictObject(W_DictMultiObject):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -293,6 +293,20 @@
else:
assert list(d) == [key] + other_keys
+ def test_delitem_if_value_is(self):
+ import __pypy__
+ class X:
+ pass
+ x2 = X()
+ x3 = X()
+ d = {2: x2, 3: x3}
+ __pypy__.delitem_if_value_is(d, 2, x3)
+ assert d == {2: x2, 3: x3}
+ __pypy__.delitem_if_value_is(d, 2, x2)
+ assert d == {3: x3}
+ __pypy__.delitem_if_value_is(d, 2, x3)
+ assert d == {3: x3}
+
def test_keys(self):
d = {1: 2, 3: 4}
kys = list(d.keys())
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -575,6 +575,10 @@
pair(self, s_key).delitem()
method_delitem_with_hash.can_only_throw = _dict_can_only_throw_keyerror
+ def method_delitem_if_value_is(self, s_key, s_value):
+ pair(self, s_key).setitem(s_value)
+ pair(self, s_key).delitem()
+
class __extend__(SomeOrderedDict):
def method_move_to_end(self, s_key, s_last):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -934,6 +934,20 @@
return
d.delitem_with_hash(key, h)
+ at specialize.call_location()
+def delitem_if_value_is(d, key, value):
+ """Same as 'if d.get(key) is value: del d[key]'. It is safe even in
+ case 'd' is an r_dict and the lookup involves callbacks that might
+ release the GIL."""
+ if not we_are_translated():
+ try:
+ if d[key] is value:
+ del d[key]
+ except KeyError:
+ pass
+ return
+ d.delitem_if_value_is(key, value)
+
def _untranslated_move_to_end(d, key, last):
"NOT_RPYTHON"
value = d.pop(key)
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -7,7 +7,7 @@
resizelist_hint, is_annotation_constant, always_inline, NOT_CONSTANT,
iterkeys_with_hash, iteritems_with_hash, contains_with_hash,
setitem_with_hash, getitem_with_hash, delitem_with_hash, import_from_mixin,
- fetch_translated_config, try_inline, move_to_end)
+ fetch_translated_config, try_inline, delitem_if_value_is, move_to_end)
from rpython.translator.translator import TranslationContext, graphof
from rpython.rtyper.test.tool import BaseRtypingTest
from rpython.rtyper.test.test_llinterp import interpret
@@ -661,6 +661,24 @@
f(29)
interpret(f, [27])
+def test_delitem_if_value_is():
+ class X:
+ pass
+ def f(i):
+ x42 = X()
+ x612 = X()
+ d = {i + .5: x42, i + .6: x612}
+ delitem_if_value_is(d, i + .5, x612)
+ assert (i + .5) in d
+ delitem_if_value_is(d, i + .5, x42)
+ assert (i + .5) not in d
+ delitem_if_value_is(d, i + .5, x612)
+ assert (i + .5) not in d
+ return 0
+
+ f(29)
+ interpret(f, [27])
+
def test_rdict_with_hash():
def f(i):
d = r_dict(strange_key_eq, strange_key_hash)
diff --git a/rpython/rtyper/lltypesystem/rordereddict.py b/rpython/rtyper/lltypesystem/rordereddict.py
--- a/rpython/rtyper/lltypesystem/rordereddict.py
+++ b/rpython/rtyper/lltypesystem/rordereddict.py
@@ -407,6 +407,12 @@
hop.exception_is_here()
hop.gendirectcall(ll_dict_delitem_with_hash, v_dict, v_key, v_hash)
+ def rtype_method_delitem_if_value_is(self, hop):
+ v_dict, v_key, v_value = hop.inputargs(
+ self, self.key_repr, self.value_repr)
+ hop.exception_cannot_occur()
+ hop.gendirectcall(ll_dict_delitem_if_value_is, v_dict, v_key, v_value)
+
def rtype_method_move_to_end(self, hop):
v_dict, v_key, v_last = hop.inputargs(
self, self.key_repr, lltype.Bool)
@@ -821,6 +827,15 @@
raise KeyError
_ll_dict_del(d, hash, index)
+def ll_dict_delitem_if_value_is(d, key, value):
+ hash = d.keyhash(key)
+ index = d.lookup_function(d, key, hash, FLAG_LOOKUP)
+ if index < 0:
+ return
+ if d.entries[index].value != value:
+ return
+ _ll_dict_del(d, hash, index)
+
def _ll_dict_del_entry(d, index):
d.entries.mark_deleted(index)
d.num_live_items -= 1
More information about the pypy-commit
mailing list