[pypy-commit] pypy default: merge strbuf-as-buffer
plan_rich
pypy.commits at gmail.com
Wed Jan 4 05:05:54 EST 2017
Author: Richard Plangger <planrichi at gmail.com>
Branch:
Changeset: r89356:8794d97c98d1
Date: 2017-01-04 10:53 +0100
http://bitbucket.org/pypy/pypy/changeset/8794d97c98d1/
Log: merge strbuf-as-buffer
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -32,7 +32,7 @@
del a; gc.collect(); gc.collect(); gc.collect()
self.assertEqual(x[:], expected)
- self.assertRaises((TypeError, ValueError),
+ self.assertRaises(TypeError,
(c_char * 16).from_buffer, "a" * 16)
def test_fom_buffer_with_offset(self):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -85,7 +85,17 @@
def from_buffer(self, obj, offset=0):
size = self._sizeofinstances()
+ if isinstance(obj, (str, unicode)):
+ # hack, buffer(str) will always return a readonly buffer.
+ # CPython calls PyObject_AsWriteBuffer(...) here!
+ # str cannot be modified, thus raise a type error in this case
+ raise TypeError("Cannot use %s as modifiable buffer" % str(type(obj)))
+
+ # why not just call memoryview(obj)[offset:]?
+ # array in Python 2.7 does not support the buffer protocol and will
+ # fail, even though buffer is supported
buf = buffer(obj, offset, size)
+
if len(buf) < size:
raise ValueError(
"Buffer size too small (%d instead of at least %d bytes)"
diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -4,6 +4,7 @@
from rpython.rlib.buffer import Buffer
from pypy.interpreter.gateway import unwrap_spec
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
class ByteBuffer(Buffer):
@@ -22,6 +23,8 @@
def setitem(self, index, char):
self.data[index] = char
+ def get_raw_address(self):
+ return nonmoving_raw_ptr_for_resizable_list(self.data)
@unwrap_spec(length=int)
def bytebuffer(space, length):
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -136,6 +136,9 @@
return _from_buffer(space, w_ctype, w_x)
def _from_buffer(space, w_ctype, w_x):
+ if space.isinstance_w(w_x, space.w_unicode):
+ raise oefmt(space.w_TypeError,
+ "from_buffer() cannot return the address a unicode")
buf = _fetch_as_read_buffer(space, w_x)
if space.isinstance_w(w_x, space.w_str):
_cdata = get_raw_address_of_string(space, w_x)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3419,24 +3419,28 @@
BCharA = new_array_type(BCharP, None)
p1 = from_buffer(BCharA, b"foo")
assert p1 == from_buffer(BCharA, b"foo")
- import gc; gc.collect()
- assert p1 == from_buffer(BCharA, b"foo")
py.test.raises(TypeError, from_buffer, BCharA, u+"foo")
try:
from __builtin__ import buffer
except ImportError:
pass
else:
- # from_buffer(buffer(b"foo")) does not work, because it's not
- # implemented on pypy; only from_buffer(b"foo") works.
- py.test.raises(TypeError, from_buffer, BCharA, buffer(b"foo"))
- py.test.raises(TypeError, from_buffer, BCharA, buffer(u+"foo"))
+ contents = from_buffer(BCharA, buffer(b"foo"))
+ for i in range(len(contents)):
+ assert contents[i] == p1[i]
+ p4 = from_buffer(BCharA, b"f\x00\x00\x00o\x00\x00\x00o\x00\x00\x00")
+ contents = from_buffer(BCharA, buffer(u+"foo"))
+ for i in range(len(contents)):
+ assert contents[i] == p4[i]
try:
from __builtin__ import memoryview
except ImportError:
pass
else:
- py.test.raises(TypeError, from_buffer, BCharA, memoryview(b"foo"))
+ contents = from_buffer(BCharA, memoryview(b"foo"))
+ for i in range(len(contents)):
+ assert contents[i] == p1[i]
+
def test_from_buffer_bytearray():
a = bytearray(b"xyz")
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -4,6 +4,7 @@
from pypy.interpreter.typedef import (
TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
from rpython.rlib.buffer import Buffer
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rarithmetic import r_longlong, intmask
@@ -120,6 +121,9 @@
def setitem(self, index, char):
self.buf[self.start + index] = char
+ def get_raw_address(self):
+ return nonmoving_raw_ptr_for_resizable_list(self.buf)
+
class BufferedMixin:
_mixin_ = True
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -424,7 +424,7 @@
w_addr = w_param3
try:
addr = self.addr_from_object(space, w_addr)
- count = self.sock.sendto(data, flags, addr)
+ count = self.sock.sendto(data, len(data), flags, addr)
except SocketError as e:
raise converted_error(space, e)
return space.wrap(count)
diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py
--- a/pypy/objspace/std/test/test_bufferobject.py
+++ b/pypy/objspace/std/test/test_bufferobject.py
@@ -199,7 +199,9 @@
raises(TypeError, "buf[MyInt(0):MyInt(5)]")
def test_pypy_raw_address_base(self):
- raises(ValueError, buffer("foobar")._pypy_raw_address)
- raises(ValueError, buffer(u"foobar")._pypy_raw_address)
- a = buffer(bytearray("foobar"))._pypy_raw_address()
+ a = buffer("foobar")._pypy_raw_address()
assert a != 0
+ b = buffer(u"foobar")._pypy_raw_address()
+ assert b != 0
+ c = buffer(bytearray("foobar"))._pypy_raw_address()
+ assert c != 0
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -56,6 +56,7 @@
assert u"abc" != memoryview("abc")
def test_pypy_raw_address_base(self):
- raises(ValueError, memoryview("foobar")._pypy_raw_address)
- a = memoryview(bytearray("foobar"))._pypy_raw_address()
+ a = memoryview("foobar")._pypy_raw_address()
assert a != 0
+ b = memoryview(bytearray("foobar"))._pypy_raw_address()
+ assert b != 0
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -158,7 +158,11 @@
# record that ignore_finalizer() has been called
GCFLAG_IGNORE_FINALIZER = first_gcflag << 10
-_GCFLAG_FIRST_UNUSED = first_gcflag << 11 # the first unused bit
+# shadow objects can have its memory initialized when it is created.
+# It does not need an additional copy in trace out
+GCFLAG_SHADOW_INITIALIZED = first_gcflag << 11
+
+_GCFLAG_FIRST_UNUSED = first_gcflag << 12 # the first unused bit
# States for the incremental GC
@@ -729,6 +733,16 @@
obj = self.external_malloc(typeid, length, alloc_young=True)
return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+ def move_out_of_nursery(self, obj):
+ # called twice, it should return the same shadow object,
+ # and not creating another shadow object
+ if self.header(obj).tid & GCFLAG_HAS_SHADOW:
+ shadow = self.nursery_objects_shadows.get(obj)
+ ll_assert(shadow != llmemory.NULL,
+ "GCFLAG_HAS_SHADOW but no shadow found")
+ return shadow
+
+ return self._allocate_shadow(obj, copy=True)
def collect(self, gen=2):
"""Do a minor (gen=0), start a major (gen=1), or do a full
@@ -1982,6 +1996,9 @@
and self.young_rawmalloced_objects.contains(obj)):
self._visit_young_rawmalloced_object(obj)
return
+ # copy the contents of the object? usually yes, but not for some
+ # shadow objects
+ copy = True
#
size_gc_header = self.gcheaderbuilder.size_gc_header
if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
@@ -2037,13 +2054,18 @@
# Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
# copied to the shadow itself.
self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+ tid = self.header(obj).tid
+ if (tid & GCFLAG_SHADOW_INITIALIZED) != 0:
+ copy = False
+ self.header(obj).tid &= ~GCFLAG_SHADOW_INITIALIZED
#
totalsize = size_gc_header + self.get_size(obj)
self.nursery_surviving_size += raw_malloc_usage(totalsize)
#
# Copy it. Note that references to other objects in the
# nursery are kept unchanged in this step.
- llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
+ if copy:
+ llmemory.raw_memcopy(obj - size_gc_header, newhdr, totalsize)
#
# Set the old object's tid to -42 (containing all flags) and
# replace the old object's content with the target address.
@@ -2570,7 +2592,8 @@
# ----------
# id() and identityhash() support
- def _allocate_shadow(self, obj):
+ @specialize.arg(2)
+ def _allocate_shadow(self, obj, copy=False):
size_gc_header = self.gcheaderbuilder.size_gc_header
size = self.get_size(obj)
shadowhdr = self._malloc_out_of_nursery(size_gc_header +
@@ -2592,6 +2615,12 @@
#
self.header(obj).tid |= GCFLAG_HAS_SHADOW
self.nursery_objects_shadows.setitem(obj, shadow)
+
+ if copy:
+ self.header(obj).tid |= GCFLAG_SHADOW_INITIALIZED
+ totalsize = size_gc_header + self.get_size(obj)
+ llmemory.raw_memcopy(obj - size_gc_header, shadow, totalsize)
+
return shadow
def _find_shadow(self, obj):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -551,6 +551,13 @@
[s_gc, SomeAddress()],
annmodel.s_None)
+ self.move_out_of_nursery_ptr = None
+ if hasattr(GCClass, 'move_out_of_nursery'):
+ self.move_out_of_nursery_ptr = getfn(GCClass.move_out_of_nursery,
+ [s_gc, SomeAddress()],
+ SomeAddress())
+
+
def create_custom_trace_funcs(self, gc, rtyper):
custom_trace_funcs = tuple(rtyper.custom_trace_funcs)
rtyper.custom_trace_funcs = custom_trace_funcs
@@ -1585,6 +1592,17 @@
hop.genop("direct_call", [self.ignore_finalizer_ptr,
self.c_const_gc, v_adr])
+ def gct_gc_move_out_of_nursery(self, hop):
+ if self.move_out_of_nursery_ptr is not None:
+ v_adr = hop.genop("cast_ptr_to_adr", [hop.spaceop.args[0]],
+ resulttype=llmemory.Address)
+ v_ret = hop.genop("direct_call", [self.move_out_of_nursery_ptr,
+ self.c_const_gc, v_adr],
+ resulttype=llmemory.Address)
+ hop.genop("cast_adr_to_ptr", [v_ret],
+ resultvar = hop.spaceop.result)
+
+
class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -2,6 +2,8 @@
Buffer protocol support.
"""
from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+ nonmoving_raw_ptr_for_resizable_list)
class Buffer(object):
@@ -84,7 +86,7 @@
def __init__(self, value):
self.value = value
- self.readonly = True
+ self.readonly = 1
def getlength(self):
return len(self.value)
@@ -108,6 +110,9 @@
return self.value[start:stop]
return Buffer.getslice(self, start, stop, step, size)
+ def get_raw_address(self):
+ from rpython.rtyper.lltypesystem import rffi
+ return rffi.get_raw_address_of_string(self.value)
class SubBuffer(Buffer):
_attrs_ = ['buffer', 'offset', 'size', 'readonly']
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -535,6 +535,25 @@
from rpython.rtyper.lltypesystem.lloperation import llop
llop.gc_ignore_finalizer(lltype.Void, obj)
+ at jit.dont_look_inside
+def move_out_of_nursery(obj):
+ """ Returns another object which is a copy of obj; but at any point
+ (either now or in the future) the returned object might suddenly
+ become identical to the one returned.
+
+ NOTE: Only use for immutable objects!
+ """
+ pass
+
+class MoveOutOfNurseryEntry(ExtRegistryEntry):
+ _about_ = move_out_of_nursery
+
+ def compute_result_annotation(self, s_obj):
+ return s_obj
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ return hop.genop('gc_move_out_of_nursery', hop.args_v, resulttype=hop.r_result)
# ____________________________________________________________
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -997,12 +997,12 @@
if signal_checker is not None:
signal_checker()
- def sendto(self, data, flags, address):
+ def sendto(self, data, length, flags, address):
"""Like send(data, flags) but allows specifying the destination
address. (Note that 'flags' is mandatory here.)"""
self.wait_for_data(True)
addr = address.lock()
- res = _c.sendto(self.fd, data, len(data), flags,
+ res = _c.sendto(self.fd, data, length, flags,
addr, address.addrlen)
address.unlock()
if res < 0:
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,4 +1,4 @@
-from rpython.rlib.buffer import *
+from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
from rpython.annotator.annrpython import RPythonAnnotator
from rpython.annotator.model import SomeInteger
@@ -64,3 +64,10 @@
for i in range(9999, 9, -1):
buf = SubBuffer(buf, 1, i)
assert buf.getlength() == 10
+
+def test_string_buffer_as_buffer():
+ buf = StringBuffer(b'hello world')
+ addr = buf.get_raw_address()
+ assert addr[0] == b'h'
+ assert addr[4] == b'o'
+ assert addr[6] == b'w'
diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py
--- a/rpython/rlib/test/test_rsocket.py
+++ b/rpython/rlib/test/test_rsocket.py
@@ -320,7 +320,7 @@
s2.bind(INETAddress('127.0.0.1', INADDR_ANY))
addr2 = s2.getsockname()
- s1.sendto('?', 0, addr2)
+ s1.sendto('?', 1, 0, addr2)
buf = s2.recv(100)
assert buf == '?'
s2.connect(addr)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -1138,6 +1138,9 @@
exc_data.exc_value = lltype.typeOf(evalue)._defl()
return bool(etype)
+ def op_gc_move_out_of_nursery(self, obj):
+ raise NotImplementedError("gc_move_out_of_nursery")
+
class Tracer(object):
Counter = 0
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -496,6 +496,8 @@
'gc_rawrefcount_to_obj': LLOp(sideeffects=False),
'gc_rawrefcount_next_dead': LLOp(),
+ 'gc_move_out_of_nursery': LLOp(),
+
# ------- JIT & GC interaction, only for some GCs ----------
'gc_adr_of_nursery_free' : LLOp(),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -739,6 +739,9 @@
def op_gc_ignore_finalizer(obj):
pass
+def op_gc_move_out_of_nursery(obj):
+ return obj
+
# ____________________________________________________________
def get_op_impl(opname):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -754,6 +754,7 @@
SIGNED = lltype.Signed
SIGNEDP = lltype.Ptr(lltype.Array(SIGNED, hints={'nolength': True}))
+
# various type mapping
# conversions between str and char*
@@ -876,6 +877,7 @@
get_nonmovingbuffer._always_inline_ = 'try' # get rid of the returned tuple
get_nonmovingbuffer._annenforceargs_ = [strtype]
+
@jit.dont_look_inside
def get_nonmovingbuffer_final_null(data):
tup = get_nonmovingbuffer(data)
@@ -1310,3 +1312,51 @@
releasegil=False,
calling_conv='c',
)
+
+
+if not we_are_translated():
+ class RawBytes(object):
+ # literal copy of _cffi_backend/func.py
+ def __init__(self, string):
+ self.ptr = str2charp(string, track_allocation=False)
+ def __del__(self):
+ free_charp(self.ptr, track_allocation=False)
+
+ TEST_RAW_ADDR_KEEP_ALIVE = {}
+
+ at jit.dont_look_inside
+def get_raw_address_of_string(string):
+ """Returns a 'char *' that is valid as long as the rpython string object is alive.
+ Two calls to to this function, given the same string parameter,
+ are guaranteed to return the same pointer.
+
+ The extra parameter key is necessary to create a weak reference.
+ The buffer of the returned pointer (if object is young) lives as long
+ as key is alive. If key goes out of scope, the buffer will eventually
+ be freed. `string` cannot go out of scope until the RawBytes object
+ referencing it goes out of scope.
+ """
+ assert isinstance(string, str)
+ from rpython.rtyper.annlowlevel import llstr
+ from rpython.rtyper.lltypesystem.rstr import STR
+ from rpython.rtyper.lltypesystem import llmemory
+ from rpython.rlib import rgc
+
+ if we_are_translated():
+ if rgc.can_move(string):
+ string = rgc.move_out_of_nursery(string)
+
+ # string cannot move now! return the address
+ lldata = llstr(string)
+ data_start = (llmemory.cast_ptr_to_adr(lldata) +
+ offsetof(STR, 'chars') +
+ llmemory.itemoffsetof(STR.chars, 0))
+ data_start = cast(CCHARP, data_start)
+ data_start[len(string)] = '\x00' # write the final extra null
+ return data_start
+ else:
+ global TEST_RAW_ADDR_KEEP_ALIVE
+ if string in TEST_RAW_ADDR_KEEP_ALIVE:
+ return TEST_RAW_ADDR_KEEP_ALIVE[string].ptr
+ TEST_RAW_ADDR_KEEP_ALIVE[string] = rb = RawBytes(string)
+ return rb.ptr
diff --git a/rpython/rtyper/lltypesystem/test/test_ztranslated.py b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/lltypesystem/test/test_ztranslated.py
@@ -0,0 +1,59 @@
+import gc
+from rpython.translator.c.test.test_genc import compile
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rlib import rgc
+
+def debug_assert(boolresult, msg):
+ if not boolresult:
+ llop.debug_print(lltype.Void, "\n\nassert failed: %s\n\n" % msg)
+ assert boolresult
+
+def use_str():
+ mystr = b'abc'
+ #debug_assert(rgc.can_move(mystr), "short string cannot move... why?")
+ ptr = rffi.get_raw_address_of_string(mystr)
+ ptr2 = rffi.get_raw_address_of_string(mystr)
+ debug_assert(ptr == ptr2, "ptr != ptr2")
+ debug_assert(ptr[0] == b'a', "notnurseryadr[0] == b'a' is is %s" % ptr[0])
+ ptr[0] = b'x' # oh no no, in real programs nobody is allowed to modify that
+ debug_assert(mystr[0] == b'a', "mystr[0] != b'a'")
+ debug_assert(ptr[0] == b'x', "notnurseryadr[0] == b'x'")
+ gc.collect()
+ nptr = rffi.get_raw_address_of_string(mystr)
+ debug_assert(nptr == ptr, "second call to mystr must return the same ptr")
+ debug_assert(ptr[0] == b'x', "failure a")
+ debug_assert(nptr[0] == b'x', "failure b")
+ mystr = None
+
+def long_str(lstr):
+ ptr = rffi.get_raw_address_of_string(lstr)
+ for i,c in enumerate(lstr):
+ debug_assert(ptr[i] == c, "failure c")
+ gc.collect()
+ ptr2 = rffi.get_raw_address_of_string(lstr)
+ debug_assert(ptr == ptr2, "ptr != ptr2!!!")
+ return ptr
+
+def main(argv=[]):
+ use_str()
+ gc.collect()
+ mystr = b"12341234aa"*4096*10
+ #debug_assert(not rgc.can_move(mystr), "long string can move... why?")
+ p1 = long_str(mystr)
+ gc.collect()
+ copystr = mystr[:]
+ copystr += 'a'
+ p2 = long_str(copystr)
+ debug_assert(p1 != p2, "p1 == p2")
+ return 0
+
+# ____________________________________________________________
+
+def target(driver, args):
+ return main
+
+def test_compiled_incminimark():
+ fn = compile(main, [], gcpolicy="incminimark")
+ fn()
More information about the pypy-commit
mailing list