[pypy-svn] r70895 - in pypy/trunk/pypy: rlib rlib/test rpython rpython/lltypesystem rpython/lltypesystem/test rpython/memory rpython/memory/gc rpython/memory/gc/test rpython/memory/gctransform rpython/memory/test rpython/test translator/c/src translator/c/test
arigo at codespeak.net
arigo at codespeak.net
Tue Jan 26 17:44:41 CET 2010
Author: arigo
Date: Tue Jan 26 17:44:40 2010
New Revision: 70895
Modified:
pypy/trunk/pypy/rlib/rgc.py
pypy/trunk/pypy/rlib/test/test_rgc.py
pypy/trunk/pypy/rpython/llinterp.py
pypy/trunk/pypy/rpython/lltypesystem/llarena.py
pypy/trunk/pypy/rpython/lltypesystem/llheap.py
pypy/trunk/pypy/rpython/lltypesystem/llmemory.py
pypy/trunk/pypy/rpython/lltypesystem/lloperation.py
pypy/trunk/pypy/rpython/lltypesystem/lltype.py
pypy/trunk/pypy/rpython/lltypesystem/opimpl.py
pypy/trunk/pypy/rpython/lltypesystem/rbuilder.py
pypy/trunk/pypy/rpython/lltypesystem/test/test_llarena.py
pypy/trunk/pypy/rpython/lltypesystem/test/test_llmemory.py
pypy/trunk/pypy/rpython/memory/gc/base.py
pypy/trunk/pypy/rpython/memory/gc/hybrid.py
pypy/trunk/pypy/rpython/memory/gc/semispace.py
pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py
pypy/trunk/pypy/rpython/memory/gctransform/boehm.py
pypy/trunk/pypy/rpython/memory/gctransform/framework.py
pypy/trunk/pypy/rpython/memory/gctransform/transform.py
pypy/trunk/pypy/rpython/memory/gcwrapper.py
pypy/trunk/pypy/rpython/memory/test/test_gc.py
pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py
pypy/trunk/pypy/rpython/test/test_rbuilder.py
pypy/trunk/pypy/translator/c/src/mem.h
pypy/trunk/pypy/translator/c/test/test_boehm.py
pypy/trunk/pypy/translator/c/test/test_newgc.py
Log:
(arigo, fijal)
Merge branch/stringbuilder2:
Simplifies a lot the implementation of stringbuilder,
getting rid of various realloc_* and resizable_buffer_*
operations. Replaces it with just one operation:
'shrink_array'.
Modified: pypy/trunk/pypy/rlib/rgc.py
==============================================================================
--- pypy/trunk/pypy/rlib/rgc.py (original)
+++ pypy/trunk/pypy/rlib/rgc.py Tue Jan 26 17:44:40 2010
@@ -242,95 +242,6 @@
hop.exception_cannot_occur()
return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype)
-def resizable_buffer_of_shape(T, init_size):
- """ Pre-allocates structure of type T (varsized) with possibility
- to reallocate it further by resize_buffer.
- """
- from pypy.rpython.lltypesystem import lltype
- return lltype.malloc(T, init_size)
-
-class ResizableBufferOfShapeEntry(ExtRegistryEntry):
- _about_ = resizable_buffer_of_shape
-
- def compute_result_annotation(self, s_T, s_init_size):
- from pypy.annotation import model as annmodel
- from pypy.rpython.lltypesystem import rffi, lltype
- assert s_T.is_constant()
- assert isinstance(s_init_size, annmodel.SomeInteger)
- T = s_T.const
- # limit ourselves to structs and to a fact that var-sized element
- # does not contain pointers.
- assert isinstance(T, lltype.Struct)
- assert isinstance(getattr(T, T._arrayfld).OF, lltype.Primitive)
- return annmodel.SomePtr(lltype.Ptr(T))
-
- def specialize_call(self, hop):
- from pypy.rpython.lltypesystem import lltype
- flags = {'flavor': 'gc'}
- vlist = [hop.inputarg(lltype.Void, 0),
- hop.inputconst(lltype.Void, flags),
- hop.inputarg(lltype.Signed, 1)]
- hop.exception_is_here()
- return hop.genop('malloc_resizable_buffer', vlist,
- resulttype=hop.r_result.lowleveltype)
-
-def resize_buffer(ptr, old_size, new_size):
- """ Resize raw buffer returned by resizable_buffer_of_shape to new size
- """
- from pypy.rpython.lltypesystem import lltype
- T = lltype.typeOf(ptr).TO
- arrayfld = T._arrayfld
- arr = getattr(ptr, arrayfld)
- # we don't have any realloc on top of cpython
- new_ptr = lltype.malloc(T, new_size)
- new_ar = getattr(new_ptr, arrayfld)
- for i in range(old_size):
- new_ar[i] = arr[i]
- return new_ptr
-
-class ResizeBufferEntry(ExtRegistryEntry):
- _about_ = resize_buffer
-
- def compute_result_annotation(self, s_ptr, s_old_size, s_new_size):
- from pypy.annotation import model as annmodel
- from pypy.rpython.lltypesystem import rffi
- assert isinstance(s_ptr, annmodel.SomePtr)
- assert isinstance(s_new_size, annmodel.SomeInteger)
- assert isinstance(s_old_size, annmodel.SomeInteger)
- return s_ptr
-
- def specialize_call(self, hop):
- from pypy.rpython.lltypesystem import lltype
- vlist = [hop.inputarg(hop.args_r[0], 0),
- hop.inputarg(lltype.Signed, 1),
- hop.inputarg(lltype.Signed, 2)]
- hop.exception_is_here()
- return hop.genop('resize_buffer', vlist,
- resulttype=hop.r_result.lowleveltype)
-
-def finish_building_buffer(ptr, final_size):
- """ Finish building resizable buffer returned by resizable_buffer_of_shape
- """
- return ptr
-
-class FinishBuildingBufferEntry(ExtRegistryEntry):
- _about_ = finish_building_buffer
-
- def compute_result_annotation(self, s_arr, s_final_size):
- from pypy.annotation.model import SomePtr, s_ImpossibleValue,\
- SomeInteger
- assert isinstance(s_arr, SomePtr)
- assert isinstance(s_final_size, SomeInteger)
- return s_arr
-
- def specialize_call(self, hop):
- from pypy.rpython.lltypesystem import lltype
- vlist = [hop.inputarg(hop.args_r[0], 0),
- hop.inputarg(hop.args_r[1], 1)]
- hop.exception_cannot_occur()
- return hop.genop('finish_building_buffer', vlist,
- resulttype=hop.r_result.lowleveltype)
-
def ll_arraycopy(source, dest, source_start, dest_start, length):
from pypy.rpython.lltypesystem.lloperation import llop
from pypy.rpython.lltypesystem import lltype, llmemory
@@ -366,6 +277,39 @@
ll_arraycopy._annspecialcase_ = 'specialize:ll'
ll_arraycopy._jit_look_inside_ = False
+def ll_shrink_array(p, smallerlength):
+ from pypy.rpython.lltypesystem.lloperation import llop
+ from pypy.rpython.lltypesystem import lltype, llmemory
+ from pypy.rlib.objectmodel import keepalive_until_here
+
+ if llop.shrink_array(lltype.Bool, p, smallerlength):
+ return p # done by the GC
+ # XXX we assume for now that the type of p is GcStruct containing a
+ # variable array, with no further pointers anywhere, and exactly one
+ # field in the fixed part -- like STR and UNICODE.
+
+ TP = lltype.typeOf(p).TO
+ newp = lltype.malloc(TP, smallerlength)
+
+ assert len(TP._names) == 2
+ field = getattr(p, TP._names[0])
+ setattr(newp, TP._names[0], field)
+
+ ARRAY = getattr(TP, TP._arrayfld)
+ offset = (llmemory.offsetof(TP, TP._arrayfld) +
+ llmemory.itemoffsetof(ARRAY, 0))
+ source_addr = llmemory.cast_ptr_to_adr(p) + offset
+ dest_addr = llmemory.cast_ptr_to_adr(newp) + offset
+ llmemory.raw_memcopy(source_addr, dest_addr,
+ llmemory.sizeof(ARRAY.OF) * smallerlength)
+
+ keepalive_until_here(p)
+ keepalive_until_here(newp)
+ return newp
+ll_shrink_array._annspecialcase_ = 'specialize:ll'
+ll_shrink_array._jit_look_inside_ = False
+
+
def no_collect(func):
func._dont_inline_ = True
func._gc_no_collect_ = True
Modified: pypy/trunk/pypy/rlib/test/test_rgc.py
==============================================================================
--- pypy/trunk/pypy/rlib/test/test_rgc.py (original)
+++ pypy/trunk/pypy/rlib/test/test_rgc.py Tue Jan 26 17:44:40 2010
@@ -56,19 +56,6 @@
assert res == True
-def test_resizable_buffer():
- from pypy.rpython.lltypesystem.rstr import STR
- from pypy.rpython.annlowlevel import hlstr
-
- def f():
- ptr = rgc.resizable_buffer_of_shape(STR, 1)
- ptr.chars[0] = 'a'
- ptr = rgc.resize_buffer(ptr, 1, 2)
- ptr.chars[1] = 'b'
- return hlstr(rgc.finish_building_buffer(ptr, 2))
-
- assert f() == 'ab'
-
def test_ll_arraycopy_1():
TYPE = lltype.GcArray(lltype.Signed)
a1 = lltype.malloc(TYPE, 10)
@@ -126,3 +113,21 @@
assert a2[i] == a1[i+2]
else:
assert a2[i] == org2[i]
+
+def test_ll_shrink_array_1():
+ py.test.skip("implement ll_shrink_array for GcStructs or GcArrays that "
+ "don't have the shape of STR or UNICODE")
+
+def test_ll_shrink_array_2():
+ S = lltype.GcStruct('S', ('x', lltype.Signed),
+ ('vars', lltype.Array(lltype.Signed)))
+ s1 = lltype.malloc(S, 5)
+ s1.x = 1234
+ for i in range(5):
+ s1.vars[i] = 50 + i
+ s2 = rgc.ll_shrink_array(s1, 3)
+ assert lltype.typeOf(s2) == lltype.Ptr(S)
+ assert s2.x == 1234
+ assert len(s2.vars) == 3
+ for i in range(3):
+ assert s2.vars[i] == 50 + i
Modified: pypy/trunk/pypy/rpython/llinterp.py
==============================================================================
--- pypy/trunk/pypy/rpython/llinterp.py (original)
+++ pypy/trunk/pypy/rpython/llinterp.py Tue Jan 26 17:44:40 2010
@@ -736,26 +736,17 @@
except MemoryError:
self.make_llexception()
- def op_malloc_nonmovable(self, obj, flags):
+ def op_malloc_nonmovable(self, TYPE, flags):
flavor = flags['flavor']
assert flavor == 'gc'
zero = flags.get('zero', False)
- return self.heap.malloc_nonmovable(obj, zero=zero)
+ return self.heap.malloc_nonmovable(TYPE, zero=zero)
- def op_malloc_nonmovable_varsize(self, obj, flags, size):
+ def op_malloc_nonmovable_varsize(self, TYPE, flags, size):
flavor = flags['flavor']
assert flavor == 'gc'
zero = flags.get('zero', False)
- return self.heap.malloc_nonmovable(obj, size, zero=zero)
-
- def op_malloc_resizable_buffer(self, obj, flags, size):
- return self.heap.malloc_resizable_buffer(obj, size)
-
- def op_resize_buffer(self, obj, old_size, new_size):
- return self.heap.resize_buffer(obj, old_size, new_size)
-
- def op_finish_building_buffer(self, obj, size):
- return self.heap.finish_building_buffer(obj, size)
+ return self.heap.malloc_nonmovable(TYPE, size, zero=zero)
def op_free(self, obj, flavor):
assert isinstance(flavor, str)
@@ -763,6 +754,9 @@
self.llinterpreter.remember_free(obj)
self.heap.free(obj, flavor=flavor)
+ def op_shrink_array(self, obj, smallersize):
+ return self.heap.shrink_array(obj, smallersize)
+
def op_zero_gc_pointers_inside(self, obj):
raise NotImplementedError("zero_gc_pointers_inside")
@@ -970,14 +964,6 @@
assert lltype.typeOf(size) == lltype.Signed
return llmemory.raw_malloc(size)
- def op_raw_realloc_grow(self, addr, old_size, size):
- assert lltype.typeOf(size) == lltype.Signed
- return llmemory.raw_realloc_grow(addr, old_size, size)
-
- def op_raw_realloc_shrink(self, addr, old_size, size):
- assert lltype.typeOf(size) == lltype.Signed
- return llmemory.raw_realloc_shrink(addr, old_size, size)
-
op_boehm_malloc = op_boehm_malloc_atomic = op_raw_malloc
def op_boehm_register_finalizer(self, p, finalizer):
Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/llarena.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/llarena.py Tue Jan 26 17:44:40 2010
@@ -103,6 +103,21 @@
Arena.object_arena_location[container] = self, offset
Arena.old_object_arena_location[container] = self, offset
+ def shrink_obj(self, offset, newsize):
+ oldbytes = self.objectsizes[offset]
+ newbytes = llmemory.raw_malloc_usage(newsize)
+ assert newbytes <= oldbytes
+ # fix self.objectsizes
+ for i in range(newbytes):
+ adr = offset + i
+ if adr in self.objectsizes:
+ assert self.objectsizes[adr] == oldbytes - i
+ self.objectsizes[adr] = newbytes - i
+ # fix self.usagemap
+ for i in range(offset + newbytes, offset + oldbytes):
+ assert self.usagemap[i] == 'x'
+ self.usagemap[i] = '#'
+
class fakearenaaddress(llmemory.fakeaddress):
def __init__(self, arena, offset):
@@ -306,6 +321,12 @@
% (addr.offset,))
addr.arena.allocate_object(addr.offset, size)
+def arena_shrink_obj(addr, newsize):
+ """ Mark object as shorter than it was
+ """
+ addr = _getfakearenaaddress(addr)
+ addr.arena.shrink_obj(addr.offset, newsize)
+
def round_up_for_allocation(size, minsize=0):
"""Round up the size in order to preserve alignment of objects
following an object. For arenas containing heterogenous objects.
@@ -460,6 +481,14 @@
llfakeimpl=arena_reserve,
sandboxsafe=True)
+def llimpl_arena_shrink_obj(addr, newsize):
+ pass
+register_external(arena_shrink_obj, [llmemory.Address, int], None,
+ 'll_arena.arena_shrink_obj',
+ llimpl=llimpl_arena_shrink_obj,
+ llfakeimpl=arena_shrink_obj,
+ sandboxsafe=True)
+
llimpl_round_up_for_allocation = rffi.llexternal('ROUND_UP_FOR_ALLOCATION',
[lltype.Signed, lltype.Signed],
lltype.Signed,
Modified: pypy/trunk/pypy/rpython/lltypesystem/llheap.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/llheap.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/llheap.py Tue Jan 26 17:44:40 2010
@@ -20,20 +20,9 @@
malloc_nonmovable = malloc
-def malloc_resizable_buffer(TP, size):
- return malloc(TP, size)
+def shrink_array(p, smallersize):
+ return False
-def resize_buffer(buf, old_size, new_size):
- ll_str = malloc(typeOf(buf).TO, new_size)
- for i in range(old_size):
- ll_str.chars[i] = buf.chars[i]
- return ll_str
-
-def finish_building_buffer(buf, final_size):
- ll_str = malloc(typeOf(buf).TO, final_size)
- for i in range(final_size):
- ll_str.chars[i] = buf.chars[i]
- return ll_str
def thread_prepare():
pass
Modified: pypy/trunk/pypy/rpython/lltypesystem/llmemory.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/llmemory.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/llmemory.py Tue Jan 26 17:44:40 2010
@@ -716,18 +716,6 @@
raise NotImplementedError(size)
return size._raw_malloc([], zero=False)
-def raw_realloc_grow(addr, old_size, size):
- new_area = size._raw_malloc([], zero=False)
- raw_memcopy(addr, new_area, old_size)
- raw_free(addr)
- return new_area
-
-def raw_realloc_shrink(addr, old_size, size):
- new_area = size._raw_malloc([], zero=False)
- raw_memcopy(addr, new_area, size)
- raw_free(addr)
- return new_area
-
def raw_free(adr):
# try to free the whole object if 'adr' is the address of the header
from pypy.rpython.memory.gcheader import GCHeaderBuilder
Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Tue Jan 26 17:44:40 2010
@@ -354,9 +354,7 @@
'malloc_varsize': LLOp(canraise=(MemoryError,), canunwindgc=True),
'malloc_nonmovable': LLOp(canraise=(MemoryError,), canunwindgc=True),
'malloc_nonmovable_varsize':LLOp(canraise=(MemoryError,),canunwindgc=True),
- 'malloc_resizable_buffer': LLOp(canraise=(MemoryError,),canunwindgc=True),
- 'resize_buffer': LLOp(canraise=(MemoryError,), canunwindgc=True),
- 'finish_building_buffer' : LLOp(canraise=(MemoryError,), canunwindgc=True),
+ 'shrink_array': LLOp(canrun=True),
'zero_gc_pointers_inside': LLOp(),
'free': LLOp(),
'getfield': LLOp(sideeffects=False, canrun=True),
@@ -391,8 +389,6 @@
'boehm_register_finalizer': LLOp(),
'boehm_disappearing_link': LLOp(),
'raw_malloc': LLOp(),
- 'raw_realloc_grow': LLOp(),
- 'raw_realloc_shrink': LLOp(),
'raw_malloc_usage': LLOp(sideeffects=False),
'raw_free': LLOp(),
'raw_memclear': LLOp(),
Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/lltype.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/lltype.py Tue Jan 26 17:44:40 2010
@@ -1528,6 +1528,9 @@
def getlength(self):
return len(self.items)
+ def shrinklength(self, newlength):
+ del self.items[newlength:]
+
def getbounds(self):
stop = len(self.items)
return 0, stop
@@ -1658,7 +1661,9 @@
def setitem(self, index, value):
assert index == 0
if value != self.array.getlength():
- raise Exception("can't change the length of an array")
+ if value > self.array.getlength():
+ raise Exception("can't grow an array in-place")
+ self.array.shrinklength(value)
def _makeptr(array, solid=False):
try:
Modified: pypy/trunk/pypy/rpython/lltypesystem/opimpl.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/opimpl.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/opimpl.py Tue Jan 26 17:44:40 2010
@@ -500,6 +500,9 @@
def op_gc_assume_young_pointers(addr):
pass
+def op_shrink_array(array, smallersize):
+ return False
+
# ____________________________________________________________
def get_op_impl(opname):
Modified: pypy/trunk/pypy/rpython/lltypesystem/rbuilder.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/rbuilder.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/rbuilder.py Tue Jan 26 17:44:40 2010
@@ -1,6 +1,6 @@
from pypy.rpython.rbuilder import AbstractStringBuilderRepr
-from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem import lltype, rstr
from pypy.rpython.lltypesystem.rstr import STR, UNICODE, char_repr,\
string_repr, unichar_repr, unicode_repr
from pypy.rpython.annlowlevel import llstr
@@ -9,42 +9,45 @@
from pypy.rpython.lltypesystem.lltype import staticAdtMethod
from pypy.tool.sourcetools import func_with_new_name
+# Think about heuristics below, maybe we can come up with something
+# better or at least compare it with list heuristics
+
GROW_FAST_UNTIL = 100*1024*1024 # 100 MB
-def new_grow_func(name):
+def new_grow_func(name, mallocfn, copycontentsfn):
def stringbuilder_grow(ll_builder, needed):
allocated = ll_builder.allocated
- if allocated < GROW_FAST_UNTIL:
- new_allocated = allocated << 1
- else:
- extra_size = allocated >> 2
- try:
- new_allocated = ovfcheck(allocated + extra_size)
- except OverflowError:
- raise MemoryError
+ #if allocated < GROW_FAST_UNTIL:
+ # new_allocated = allocated << 1
+ #else:
+ extra_size = allocated >> 2
try:
+ new_allocated = ovfcheck(allocated + extra_size)
new_allocated = ovfcheck(new_allocated + needed)
except OverflowError:
raise MemoryError
- ll_builder.buf = rgc.resize_buffer(ll_builder.buf, ll_builder.used,
- new_allocated)
+ newbuf = mallocfn(new_allocated)
+ copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+ ll_builder.buf = newbuf
ll_builder.allocated = new_allocated
return func_with_new_name(stringbuilder_grow, name)
-stringbuilder_grow = new_grow_func('stringbuilder_grow')
-unicodebuilder_grow = new_grow_func('unicodebuilder_grow')
+stringbuilder_grow = new_grow_func('stringbuilder_grow', rstr.mallocstr,
+ rstr.copy_string_contents)
+unicodebuilder_grow = new_grow_func('unicodebuilder_grow', rstr.mallocunicode,
+ rstr.copy_unicode_contents)
STRINGBUILDER = lltype.GcStruct('stringbuilder',
- ('allocated', lltype.Signed),
- ('used', lltype.Signed),
- ('buf', lltype.Ptr(STR)),
- adtmeths={'grow':staticAdtMethod(stringbuilder_grow)})
+ ('allocated', lltype.Signed),
+ ('used', lltype.Signed),
+ ('buf', lltype.Ptr(STR)),
+ adtmeths={'grow':staticAdtMethod(stringbuilder_grow)})
UNICODEBUILDER = lltype.GcStruct('unicodebuilder',
('allocated', lltype.Signed),
('used', lltype.Signed),
('buf', lltype.Ptr(UNICODE)),
- adtmeths={'grow':staticAdtMethod(unicodebuilder_grow)})
+ adtmeths={'grow':staticAdtMethod(unicodebuilder_grow)})
MAX = 16*1024*1024
@@ -56,7 +59,7 @@
ll_builder = lltype.malloc(cls.lowleveltype.TO)
ll_builder.allocated = init_size
ll_builder.used = 0
- ll_builder.buf = rgc.resizable_buffer_of_shape(cls.basetp, init_size)
+ ll_builder.buf = cls.mallocfn(init_size)
return ll_builder
@staticmethod
@@ -64,7 +67,7 @@
used = ll_builder.used
lgt = len(ll_str.chars)
needed = lgt + used
- if needed >= ll_builder.allocated:
+ if needed > ll_builder.allocated:
ll_builder.grow(ll_builder, lgt)
ll_str.copy_contents(ll_str, ll_builder.buf, 0, used, lgt)
ll_builder.used = needed
@@ -80,7 +83,7 @@
def ll_append_slice(ll_builder, ll_str, start, end):
needed = end - start
used = ll_builder.used
- if needed + used >= ll_builder.allocated:
+ if needed + used > ll_builder.allocated:
ll_builder.grow(ll_builder, needed)
assert needed >= 0
ll_str.copy_contents(ll_str, ll_builder.buf, start, used, needed)
@@ -89,7 +92,7 @@
@staticmethod
def ll_append_multiple_char(ll_builder, char, times):
used = ll_builder.used
- if times + used >= ll_builder.allocated:
+ if times + used > ll_builder.allocated:
ll_builder.grow(ll_builder, times)
for i in range(times):
ll_builder.buf.chars[used] = char
@@ -99,17 +102,22 @@
@staticmethod
def ll_build(ll_builder):
final_size = ll_builder.used
- return rgc.finish_building_buffer(ll_builder.buf, final_size)
+ assert final_size >= 0
+ if final_size == ll_builder.allocated:
+ return ll_builder.buf
+ return rgc.ll_shrink_array(ll_builder.buf, final_size)
class StringBuilderRepr(BaseStringBuilderRepr):
lowleveltype = lltype.Ptr(STRINGBUILDER)
basetp = STR
+ mallocfn = staticmethod(rstr.mallocstr)
string_repr = string_repr
char_repr = char_repr
class UnicodeBuilderRepr(BaseStringBuilderRepr):
lowleveltype = lltype.Ptr(UNICODEBUILDER)
basetp = UNICODE
+ mallocfn = staticmethod(rstr.mallocunicode)
string_repr = unicode_repr
char_repr = unichar_repr
Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_llarena.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/test/test_llarena.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/test/test_llarena.py Tue Jan 26 17:44:40 2010
@@ -5,6 +5,7 @@
from pypy.rpython.lltypesystem.llarena import arena_reserve, arena_free
from pypy.rpython.lltypesystem.llarena import round_up_for_allocation
from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view
+from pypy.rpython.lltypesystem.llarena import arena_shrink_obj
def test_arena():
S = lltype.Struct('S', ('x',lltype.Signed))
@@ -268,3 +269,16 @@
fn = compile(test_look_inside_object, [])
res = fn()
assert res == 42
+
+def test_shrink_obj():
+ from pypy.rpython.memory.gcheader import GCHeaderBuilder
+ HDR = lltype.Struct('HDR', ('h', lltype.Signed))
+ gcheaderbuilder = GCHeaderBuilder(HDR)
+ size_gc_header = gcheaderbuilder.size_gc_header
+ S = lltype.GcStruct('S', ('x', lltype.Signed),
+ ('a', lltype.Array(lltype.Signed)))
+ myarenasize = 200
+ a = arena_malloc(myarenasize, False)
+ arena_reserve(a, size_gc_header + llmemory.sizeof(S, 10))
+ arena_shrink_obj(a, size_gc_header + llmemory.sizeof(S, 5))
+ arena_reset(a, size_gc_header + llmemory.sizeof(S, 5), False)
Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_llmemory.py
==============================================================================
--- pypy/trunk/pypy/rpython/lltypesystem/test/test_llmemory.py (original)
+++ pypy/trunk/pypy/rpython/lltypesystem/test/test_llmemory.py Tue Jan 26 17:44:40 2010
@@ -624,31 +624,3 @@
# the following line crashes if the array is dead
ptr1 = cast_adr_to_ptr(adr, lltype.Ptr(lltype.FixedSizeArray(Address, 1)))
ptr1[0] = NULL
-
-def test_realloc():
- A = lltype.Array(lltype.Float)
- adr = raw_malloc(sizeof(A, 10))
- ptr = cast_adr_to_ptr(adr, lltype.Ptr(A))
- for i in range(10):
- ptr[i] = float(i)
- adr2 = raw_realloc_shrink(adr, sizeof(A, 10), sizeof(A, 5))
- ptr2 = cast_adr_to_ptr(adr2, lltype.Ptr(A))
- assert len(ptr2) == 5
- assert ptr2[3] == 3.0
- assert ptr2[1] == 1.0
-
-def test_realloc_struct():
- S = lltype.Struct('x', ('one', lltype.Signed),
- ('a', lltype.Array(lltype.Float)))
- adr = raw_malloc(sizeof(S, 5))
- ptr = cast_adr_to_ptr(adr, lltype.Ptr(S))
- for i in range(5):
- ptr.a[i] = float(i)
- ptr.one = 3
- adr2 = raw_realloc_grow(adr, sizeof(S, 5), sizeof(S, 10))
- ptr2 = cast_adr_to_ptr(adr2, lltype.Ptr(S))
- assert len(ptr2.a) == 10
- assert ptr2.a[3] == 3.0
- assert ptr2.a[0] == 0.0
- assert ptr2.one == 3
-
Modified: pypy/trunk/pypy/rpython/memory/gc/base.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gc/base.py (original)
+++ pypy/trunk/pypy/rpython/memory/gc/base.py Tue Jan 26 17:44:40 2010
@@ -17,7 +17,6 @@
needs_write_barrier = False
malloc_zero_filled = False
prebuilt_gc_objects_are_static_roots = True
- can_realloc = False
object_minimal_size = 0
def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE):
Modified: pypy/trunk/pypy/rpython/memory/gc/hybrid.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gc/hybrid.py (original)
+++ pypy/trunk/pypy/rpython/memory/gc/hybrid.py Tue Jan 26 17:44:40 2010
@@ -31,7 +31,6 @@
# Object lists:
# * gen2_rawmalloced_objects
# * gen3_rawmalloced_objects
-# * gen2_resizable_objects
# * old_objects_pointing_to_young: gen2or3 objs that point to gen1 objs
# * last_generation_root_objects: gen3 objs that point to gen1or2 objs
#
@@ -44,15 +43,11 @@
# Some invariants:
# * gen3 are either GCFLAG_NO_HEAP_PTRS or in 'last_generation_root_objects'
# * between collections, GCFLAG_UNVISITED set exactly for gen2_rawmalloced
-# * objects in gen2_resizable_objects are part of the generation 2 but never
-# explicitly listed in gen2_rawmalloced_objects.
#
# A malloc_varsize() of large objects returns objects that are external
# but initially of generation 2. Old objects from the semispaces are
# moved to external objects directly as generation 3.
-# gen2_resizable_objects is for objects that are resizable
-
# The "age" of an object is the number of times it survived a full
# collections, without counting the step that moved it out of the nursery.
# When a semispace-based object would grow older than MAX_SEMISPACE_AGE,
@@ -82,7 +77,6 @@
"""
first_unused_gcflag = _gcflag_next_bit
prebuilt_gc_objects_are_static_roots = True
- can_realloc = False
# the following values override the default arguments of __init__ when
# translating to a real backend.
@@ -124,7 +118,6 @@
self.gen2_rawmalloced_objects = self.AddressStack()
self.gen3_rawmalloced_objects = self.AddressStack()
- self.gen2_resizable_objects = self.AddressStack()
GenerationGC.setup(self)
def set_max_heap_size(self, size):
@@ -176,8 +169,7 @@
llmemory.GCREF)
return self.malloc_varsize_slowpath(typeid, length)
- def malloc_varsize_slowpath(self, typeid, length, force_nonmovable=False,
- resizable=False):
+ def malloc_varsize_slowpath(self, typeid, length, force_nonmovable=False):
# For objects that are too large, or when the nursery is exhausted.
# In order to keep malloc_varsize_clear() as compact as possible,
# we recompute what we need in this slow path instead of passing
@@ -196,7 +188,7 @@
else:
nonlarge_max = self.nonlarge_max
if force_nonmovable or raw_malloc_usage(totalsize) > nonlarge_max:
- result = self.malloc_varsize_marknsweep(totalsize, resizable)
+ result = self.malloc_varsize_marknsweep(totalsize)
flags = self.GCFLAGS_FOR_NEW_EXTERNAL_OBJECTS | GCFLAG_UNVISITED
else:
result = self.malloc_varsize_collecting_nursery(totalsize)
@@ -210,9 +202,6 @@
def malloc_varsize_nonmovable(self, typeid, length):
return self.malloc_varsize_slowpath(typeid, length, True)
- def malloc_varsize_resizable(self, typeid, length):
- return self.malloc_varsize_slowpath(typeid, length, True, True)
-
def malloc_nonmovable(self, typeid, length, zero):
# helper for testing, same as GCBase.malloc
if self.is_varsize(typeid):
@@ -221,40 +210,6 @@
raise NotImplementedError("Not supported")
return llmemory.cast_ptr_to_adr(gcref)
- def realloc(self, ptr, newlength, fixedsize, itemsize, lengthofs, grow):
- size_gc_header = self.size_gc_header()
- addr = llmemory.cast_ptr_to_adr(ptr)
- ll_assert(self.header(addr).tid & GCFLAG_EXTERNAL,
- "realloc() on a non-external object")
- nonvarsize = size_gc_header + fixedsize
- try:
- varsize = ovfcheck(itemsize * newlength)
- tot_size = ovfcheck(nonvarsize + varsize)
- except OverflowError:
- raise MemoryError()
- oldlength = (addr + lengthofs).signed[0]
- old_tot_size = size_gc_header + fixedsize + oldlength * itemsize
- source_addr = addr - size_gc_header
- self.gen2_resizable_objects.remove(addr)
- if grow:
- result = llop.raw_realloc_grow(llmemory.Address, source_addr,
- old_tot_size, tot_size)
- else:
- result = llop.raw_realloc_shrink(llmemory.Address, source_addr,
- old_tot_size, tot_size)
- if not result:
- self.gen2_resizable_objects.append(addr)
- raise MemoryError()
- if grow:
- self.gen2_resizable_objects.append(result + size_gc_header)
- else:
- self.gen2_rawmalloced_objects.append(result + size_gc_header)
- self._check_rawsize_alloced(raw_malloc_usage(tot_size) -
- raw_malloc_usage(old_tot_size),
- can_collect = not grow)
- (result + size_gc_header + lengthofs).signed[0] = newlength
- return llmemory.cast_adr_to_ptr(result + size_gc_header, llmemory.GCREF)
-
def can_move(self, addr):
tid = self.header(addr).tid
return not (tid & GCFLAG_EXTERNAL)
@@ -278,7 +233,7 @@
self.semispace_collect()
debug_stop("gc-rawsize-collect")
- def malloc_varsize_marknsweep(self, totalsize, resizable=False):
+ def malloc_varsize_marknsweep(self, totalsize):
# In order to free the large objects from time to time, we
# arbitrarily force a full collect() if none occurs when we have
# allocated 'self.space_size' bytes of large objects.
@@ -294,10 +249,7 @@
# need to follow suit.
llmemory.raw_memclear(result, totalsize)
size_gc_header = self.gcheaderbuilder.size_gc_header
- if resizable:
- self.gen2_resizable_objects.append(result + size_gc_header)
- else:
- self.gen2_rawmalloced_objects.append(result + size_gc_header)
+ self.gen2_rawmalloced_objects.append(result + size_gc_header)
return result
def allocate_external_object(self, totalsize):
@@ -469,7 +421,6 @@
if self.is_collecting_gen3():
self.sweep_rawmalloced_objects(generation=3)
self.sweep_rawmalloced_objects(generation=2)
- self.sweep_rawmalloced_objects(generation=-2)
# As we just collected, it's fine to raw_malloc'ate up to space_size
# bytes again before we should force another collect.
self.large_objects_collect_trigger = self.space_size
@@ -500,10 +451,8 @@
gen3roots.delete()
self.last_generation_root_objects = newgen3roots
else:
- # mostly a hack: the generation number -2 is the part of the
- # generation 2 that lives in gen2_resizable_objects
- ll_assert(generation == -2, "bogus 'generation'")
- objects = self.gen2_resizable_objects
+ ll_assert(False, "bogus 'generation'")
+ return
surviving_objects = self.AddressStack()
# Help the flow space
@@ -538,18 +487,11 @@
tid |= GCFLAG_UNVISITED
surviving_objects.append(obj)
self.header(obj).tid = tid
- elif generation == -2:
- # the object stays in generation -2
- tid |= GCFLAG_UNVISITED
- surviving_objects.append(obj)
- self.header(obj).tid = tid
objects.delete()
if generation == 2:
self.gen2_rawmalloced_objects = surviving_objects
elif generation == 3:
self.gen3_rawmalloced_objects = surviving_objects
- elif generation == -2:
- self.gen2_resizable_objects = surviving_objects
debug_print("| [hyb] gen", generation,
"nonmoving now alive: ",
alive_size, "bytes in",
Modified: pypy/trunk/pypy/rpython/memory/gc/semispace.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gc/semispace.py (original)
+++ pypy/trunk/pypy/rpython/memory/gc/semispace.py Tue Jan 26 17:44:40 2010
@@ -119,6 +119,21 @@
self.free = result + llarena.round_up_for_allocation(totalsize)
return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
+ def shrink_array(self, addr, smallerlength):
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ if self._is_in_the_space(addr - size_gc_header):
+ typeid = self.get_type_id(addr)
+ totalsmallersize = (
+ size_gc_header + self.fixed_size(typeid) +
+ self.varsize_item_sizes(typeid) * smallerlength)
+ llarena.arena_shrink_obj(addr - size_gc_header, totalsmallersize)
+ #
+ offset_to_length = self.varsize_offset_to_length(typeid)
+ (addr + offset_to_length).signed[0] = smallerlength
+ return True
+ else:
+ return False
+
def obtain_free_space(self, needed):
# a bit of tweaking to maximize the performance and minimize the
# amount of code in an inlined version of malloc_fixedsize_clear()
@@ -572,6 +587,9 @@
def _is_external(self, obj):
return (self.header(obj).tid & GCFLAG_EXTERNAL) != 0
+ def _is_in_the_space(self, obj):
+ return self.tospace <= obj < self.free
+
def debug_check_object(self, obj):
"""Check the invariants about 'obj' that should be true
between collections."""
Modified: pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py (original)
+++ pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py Tue Jan 26 17:44:40 2010
@@ -268,6 +268,22 @@
class TestSemiSpaceGC(DirectGCTest):
from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
+ def test_shrink_array(self):
+ S1 = lltype.GcStruct('S1', ('h', lltype.Char),
+ ('v', lltype.Array(lltype.Char)))
+ p1 = self.malloc(S1, 2)
+ p1.h = '?'
+ for i in range(2):
+ p1.v[i] = chr(50 + i)
+ addr = llmemory.cast_ptr_to_adr(p1)
+ ok = self.gc.shrink_array(addr, 1)
+ assert ok
+ assert p1.h == '?'
+ assert len(p1.v) == 1
+ for i in range(1):
+ assert p1.v[i] == chr(50 + i)
+
+
class TestGenerationGC(TestSemiSpaceGC):
from pypy.rpython.memory.gc.generation import GenerationGC as GCClass
@@ -358,7 +374,7 @@
gc.collect(9)
assert calls == [('semispace_collect', True)]
- calls = []
+ calls = []
class TestMarkCompactGC(DirectGCTest):
Modified: pypy/trunk/pypy/rpython/memory/gctransform/boehm.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gctransform/boehm.py (original)
+++ pypy/trunk/pypy/rpython/memory/gctransform/boehm.py Tue Jan 26 17:44:40 2010
@@ -22,19 +22,12 @@
mh = mallocHelpers()
mh.allocate = lambda size: llop.boehm_malloc(llmemory.Address, size)
- c_realloc = rffi.llexternal('GC_REALLOC', [rffi.VOIDP, rffi.INT],
- rffi.VOIDP, sandboxsafe=True)
- def _realloc(ptr, size):
- return llmemory.cast_ptr_to_adr(c_realloc(rffi.cast(rffi.VOIDP, ptr), size))
- mh.realloc = _realloc
ll_malloc_fixedsize = mh._ll_malloc_fixedsize
# XXX, do we need/want an atomic version of this function?
ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length
ll_malloc_varsize = mh.ll_malloc_varsize
- ll_realloc = mh.ll_realloc
-
HDRPTR = lltype.Ptr(self.HDR)
def ll_identityhash(addr):
@@ -58,9 +51,6 @@
inline=False)
self.weakref_deref_ptr = self.inittime_helper(
ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address)
- self.realloc_ptr = self.inittime_helper(
- ll_realloc, [llmemory.Address] + [lltype.Signed] * 4,
- llmemory.Address)
self.identityhash_ptr = self.inittime_helper(
ll_identityhash, [llmemory.Address], lltype.Signed,
inline=False)
@@ -73,15 +63,6 @@
def pop_alive_nopyobj(self, var, llops):
pass
- def _can_realloc(self):
- return True
-
- def perform_realloc(self, hop, v_ptr, v_newlgt, c_const_size, c_item_size,
- c_lengthofs, c_grow):
- args = [self.realloc_ptr, v_ptr, v_newlgt, c_const_size,
- c_item_size, c_lengthofs]
- return hop.genop('direct_call', args, resulttype=llmemory.Address)
-
def gct_fv_gc_malloc(self, hop, flags, TYPE, c_size):
# XXX same behavior for zero=True: in theory that's wrong
if TYPE._is_atomic():
Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original)
+++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Tue Jan 26 17:44:40 2010
@@ -276,6 +276,14 @@
[s_gc, annmodel.SomeAddress()],
annmodel.SomeBool())
+ if hasattr(GCClass, 'shrink_array'):
+ self.shrink_array_ptr = getfn(
+ GCClass.shrink_array.im_func,
+ [s_gc, annmodel.SomeAddress(),
+ annmodel.SomeInteger(nonneg=True)], annmodel.s_Bool)
+ else:
+ self.shrink_array_ptr = None
+
if hasattr(GCClass, 'assume_young_pointers'):
# xxx should really be a noop for gcs without generations
self.assume_young_pointers_ptr = getfn(
@@ -358,25 +366,6 @@
else:
self.malloc_varsize_nonmovable_ptr = None
- if getattr(GCClass, 'malloc_varsize_resizable', False):
- malloc_resizable = func_with_new_name(
- GCClass.malloc_varsize_resizable.im_func,
- "malloc_varsize_resizable")
- self.malloc_varsize_resizable_ptr = getfn(
- malloc_resizable,
- [s_gc, s_typeid16,
- annmodel.SomeInteger(nonneg=True)], s_gcref)
- else:
- self.malloc_varsize_resizable_ptr = None
-
- if getattr(GCClass, 'realloc', False):
- self.realloc_ptr = getfn(
- GCClass.realloc.im_func,
- [s_gc, s_gcref] +
- [annmodel.SomeInteger(nonneg=True)] * 4 +
- [annmodel.SomeBool()],
- s_gcref)
-
self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
[s_gc, s_gcref],
annmodel.SomeInteger(),
@@ -627,11 +616,7 @@
info_varsize.ofstolength)
c_varitemsize = rmodel.inputconst(lltype.Signed,
info_varsize.varitemsize)
- if flags.get('resizable') and self.malloc_varsize_resizable_ptr:
- assert c_can_collect.value
- malloc_ptr = self.malloc_varsize_resizable_ptr
- args = [self.c_const_gc, c_type_id, v_length]
- elif flags.get('nonmovable') and self.malloc_varsize_nonmovable_ptr:
+ if flags.get('nonmovable') and self.malloc_varsize_nonmovable_ptr:
# we don't have tests for such cases, let's fail
# explicitely
assert c_can_collect.value
@@ -673,6 +658,17 @@
hop.genop("direct_call", [self.can_move_ptr, self.c_const_gc, v_addr],
resultvar=op.result)
+ def gct_shrink_array(self, hop):
+ if self.shrink_array_ptr is None:
+ return GCTransformer.gct_shrink_array(self, hop)
+ op = hop.spaceop
+ v_addr = hop.genop('cast_ptr_to_adr',
+ [op.args[0]], resulttype=llmemory.Address)
+ v_length = op.args[1]
+ hop.genop("direct_call", [self.shrink_array_ptr, self.c_const_gc,
+ v_addr, v_length],
+ resultvar=op.result)
+
def gct_gc_assume_young_pointers(self, hop):
op = hop.spaceop
v_addr = op.args[0]
@@ -716,19 +712,6 @@
resulttype=llmemory.Address)
hop.genop('adr_add', [v_gc_adr, c_ofs], resultvar=op.result)
- def _can_realloc(self):
- return self.gcdata.gc.can_realloc
-
- def perform_realloc(self, hop, v_ptr, v_newsize, c_const_size,
- c_itemsize, c_lengthofs, c_grow):
- vlist = [self.realloc_ptr, self.c_const_gc, v_ptr, v_newsize,
- c_const_size, c_itemsize, c_lengthofs, c_grow]
- livevars = self.push_roots(hop)
- v_result = hop.genop('direct_call', vlist,
- resulttype=llmemory.GCREF)
- self.pop_roots(hop, livevars)
- return v_result
-
def gct_gc_x_swap_pool(self, hop):
op = hop.spaceop
[v_malloced] = op.args
Modified: pypy/trunk/pypy/rpython/memory/gctransform/transform.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gctransform/transform.py (original)
+++ pypy/trunk/pypy/rpython/memory/gctransform/transform.py Tue Jan 26 17:44:40 2010
@@ -482,15 +482,6 @@
return result
mh.ll_malloc_varsize_no_length_zero = _ll_malloc_varsize_no_length_zero
- def ll_realloc(ptr, length, constsize, itemsize, lengthoffset):
- size = constsize + length * itemsize
- result = mh.realloc(ptr, size)
- if not result:
- raise MemoryError()
- (result + lengthoffset).signed[0] = length
- return result
- mh.ll_realloc = ll_realloc
-
return mh
class GCTransformer(BaseGCTransformer):
@@ -563,81 +554,6 @@
def gct_malloc_nonmovable_varsize(self, *args, **kwds):
return self.gct_malloc_varsize(*args, **kwds)
- def gct_malloc_resizable_buffer(self, hop):
- flags = hop.spaceop.args[1].value
- flags['varsize'] = True
- flags['nonmovable'] = True
- flags['resizable'] = True
- flavor = flags['flavor']
- assert flavor != 'cpy', "cannot malloc CPython objects directly"
- meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None)
- assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor)
- return self.varsize_malloc_helper(hop, flags, meth, [])
-
- def gct_resize_buffer(self, hop):
- op = hop.spaceop
- if self._can_realloc():
- self._gct_resize_buffer_realloc(hop, op.args[2], True)
- else:
- self._gct_resize_buffer_no_realloc(hop, op.args[1])
-
- def _can_realloc(self):
- return False
-
- def _gct_resize_buffer_realloc(self, hop, v_newsize, grow=True):
- def intconst(c): return rmodel.inputconst(lltype.Signed, c)
- op = hop.spaceop
- flags = {'flavor':'gc', 'varsize': True}
- TYPE = op.args[0].concretetype.TO
- ARRAY = TYPE._flds[TYPE._arrayfld]
- offset_to_length = llmemory.FieldOffset(TYPE, TYPE._arrayfld) + \
- llmemory.ArrayLengthOffset(ARRAY)
- c_const_size = intconst(llmemory.sizeof(TYPE, 0))
- c_item_size = intconst(llmemory.sizeof(ARRAY.OF))
-
- c_lengthofs = intconst(offset_to_length)
- v_ptr = op.args[0]
- v_ptr = gen_cast(hop.llops, llmemory.GCREF, v_ptr)
- c_grow = rmodel.inputconst(lltype.Bool, grow)
- v_raw = self.perform_realloc(hop, v_ptr, v_newsize, c_const_size,
- c_item_size, c_lengthofs, c_grow)
- hop.cast_result(v_raw)
-
- def _gct_resize_buffer_no_realloc(self, hop, v_lgt):
- op = hop.spaceop
- meth = self.gct_fv_gc_malloc_varsize
- flags = {'flavor':'gc', 'varsize': True, 'keep_current_args': True}
- self.varsize_malloc_helper(hop, flags, meth, [])
- # fish resvar
- v_newbuf = hop.llops[-1].result
- v_src = op.args[0]
- TYPE = v_src.concretetype.TO
- c_fldname = rmodel.inputconst(lltype.Void, TYPE._arrayfld)
- v_adrsrc = hop.genop('cast_ptr_to_adr', [v_src],
- resulttype=llmemory.Address)
- v_adrnewbuf = hop.genop('cast_ptr_to_adr', [v_newbuf],
- resulttype=llmemory.Address)
- ofs = (llmemory.offsetof(TYPE, TYPE._arrayfld) +
- llmemory.itemoffsetof(getattr(TYPE, TYPE._arrayfld), 0))
- v_ofs = rmodel.inputconst(lltype.Signed, ofs)
- v_adrsrc = hop.genop('adr_add', [v_adrsrc, v_ofs],
- resulttype=llmemory.Address)
- v_adrnewbuf = hop.genop('adr_add', [v_adrnewbuf, v_ofs],
- resulttype=llmemory.Address)
- size = llmemory.sizeof(getattr(TYPE, TYPE._arrayfld).OF)
- c_size = rmodel.inputconst(lltype.Signed, size)
- v_lgtsym = hop.genop('int_mul', [c_size, v_lgt],
- resulttype=lltype.Signed)
- vlist = [v_adrsrc, v_adrnewbuf, v_lgtsym]
- hop.genop('raw_memcopy', vlist)
-
- def gct_finish_building_buffer(self, hop):
- op = hop.spaceop
- if self._can_realloc():
- return self._gct_resize_buffer_realloc(hop, op.args[1], False)
- else:
- return self._gct_resize_buffer_no_realloc(hop, op.args[1])
-
def varsize_malloc_helper(self, hop, flags, meth, extraargs):
def intconst(c): return rmodel.inputconst(lltype.Signed, c)
op = hop.spaceop
@@ -698,3 +614,6 @@
def gct_gc_can_move(self, hop):
return hop.cast_result(rmodel.inputconst(lltype.Bool, False))
+
+ def gct_shrink_array(self, hop):
+ return hop.cast_result(rmodel.inputconst(lltype.Bool, False))
Modified: pypy/trunk/pypy/rpython/memory/gcwrapper.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/gcwrapper.py (original)
+++ pypy/trunk/pypy/rpython/memory/gcwrapper.py Tue Jan 26 17:44:40 2010
@@ -4,7 +4,6 @@
from pypy.rpython.memory import gctypelayout
from pypy.objspace.flow.model import Constant
-
class GCManagedHeap(object):
def __init__(self, llinterp, flowgraphs, gc_class, GC_PARAMS={}):
@@ -59,27 +58,11 @@
gctypelayout.zero_gc_pointers(result)
return result
- def malloc_resizable_buffer(self, TYPE, n):
- typeid = self.get_type_id(TYPE)
- addr = self.gc.malloc(typeid, n)
- result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE))
- if not self.gc.malloc_zero_filled:
- gctypelayout.zero_gc_pointers(result)
- return result
-
- def resize_buffer(self, obj, old_size, new_size):
- T = lltype.typeOf(obj).TO
- buf = self.malloc_resizable_buffer(T, new_size)
- # copy contents
- arrayfld = T._arrayfld
- new_arr = getattr(buf, arrayfld)
- old_arr = getattr(obj, arrayfld)
- for i in range(old_size):
- new_arr[i] = old_arr[i]
- return buf
-
- def finish_building_buffer(self, obj, size):
- return obj
+ def shrink_array(self, p, smallersize):
+ if hasattr(self.gc, 'shrink_array'):
+ addr = llmemory.cast_ptr_to_adr(p)
+ return self.gc.shrink_array(addr, smallersize)
+ return False
def free(self, TYPE, flavor='gc'):
assert flavor != 'gc'
Modified: pypy/trunk/pypy/rpython/memory/test/test_gc.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/test/test_gc.py (original)
+++ pypy/trunk/pypy/rpython/memory/test/test_gc.py Tue Jan 26 17:44:40 2010
@@ -11,7 +11,7 @@
from pypy.rlib.objectmodel import we_are_translated
from pypy.rlib.objectmodel import compute_unique_id, keepalive_until_here
from pypy.rlib import rgc
-
+from pypy.rlib.rstring import StringBuilder
def stdout_ignore_ll_functions(msg):
strmsg = str(msg)
@@ -24,6 +24,7 @@
GC_PARAMS = {}
GC_CAN_MOVE = False
GC_CANNOT_MALLOC_NONMOVABLE = False
+ GC_CAN_SHRINK_ARRAY = False
def setup_class(cls):
cls._saved_logstate = py.log._getstate()
@@ -468,18 +469,30 @@
assert self.interpret(func, []) == int(self.GC_CANNOT_MALLOC_NONMOVABLE)
- def test_resizable_buffer(self):
+ def test_shrink_array(self):
from pypy.rpython.lltypesystem.rstr import STR
- from pypy.rpython.annlowlevel import hlstr
-
- def f():
- ptr = rgc.resizable_buffer_of_shape(STR, 1)
- ptr.chars[0] = 'a'
- ptr = rgc.resize_buffer(ptr, 1, 2)
- ptr.chars[1] = 'b'
- return len(hlstr(rgc.finish_building_buffer(ptr, 2)))
+ GC_CAN_SHRINK_ARRAY = self.GC_CAN_SHRINK_ARRAY
- assert self.interpret(f, []) == 2
+ def f(n, m):
+ ptr = lltype.malloc(STR, n)
+ ptr.hash = 0x62
+ ptr.chars[0] = 'A'
+ ptr.chars[1] = 'B'
+ ptr.chars[2] = 'C'
+ ptr2 = rgc.ll_shrink_array(ptr, 2)
+ assert (ptr == ptr2) == GC_CAN_SHRINK_ARRAY
+ rgc.collect()
+ return ( ord(ptr2.chars[0]) +
+ (ord(ptr2.chars[1]) << 8) +
+ (len(ptr2.chars) << 16) +
+ (ptr2.hash << 24))
+
+ assert self.interpret(f, [3, 0]) == 0x62024241
+ # don't test with larger numbers of top of the Hybrid GC, because
+ # the default settings make it a too-large varsized object that
+ # gets allocated outside the semispace
+ if not isinstance(self, TestHybridGC):
+ assert self.interpret(f, [12, 0]) == 0x62024241
def test_tagged_simple(self):
from pypy.rlib.objectmodel import UnboxedValue
@@ -571,6 +584,20 @@
self.interpret(fn, [])
+ def test_stringbuilder(self):
+ def fn():
+ s = StringBuilder(4)
+ s.append("abcd")
+ s.append("defg")
+ s.append("rty")
+ s.append_multiple_char('y', 1000)
+ rgc.collect()
+ s.append_multiple_char('y', 1000)
+ res = s.build()[1000]
+ rgc.collect()
+ return ord(res)
+ res = self.interpret(fn, [])
+ assert res == ord('y')
from pypy.rlib.objectmodel import UnboxedValue
@@ -599,6 +626,7 @@
from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
GC_CAN_MOVE = True
GC_CANNOT_MALLOC_NONMOVABLE = True
+ GC_CAN_SHRINK_ARRAY = True
class TestGrowingSemiSpaceGC(TestSemiSpaceGC):
GC_PARAMS = {'space_size': 64}
Modified: pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py
==============================================================================
--- pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py (original)
+++ pypy/trunk/pypy/rpython/memory/test/test_transformed_gc.py Tue Jan 26 17:44:40 2010
@@ -162,6 +162,7 @@
return run
class GenericGCTests(GCTest):
+ GC_CAN_SHRINK_ARRAY = False
def heap_usage(self, statistics):
try:
@@ -631,22 +632,30 @@
run = self.runner("malloc_nonmovable_fixsize")
assert run([]) == int(self.GC_CANNOT_MALLOC_NONMOVABLE)
- def define_resizable_buffer(cls):
+ def define_shrink_array(cls):
from pypy.rpython.lltypesystem.rstr import STR
- from pypy.rpython.annlowlevel import hlstr
def f():
- ptr = rgc.resizable_buffer_of_shape(STR, 2)
- ptr.chars[0] = 'a'
- ptr = rgc.resize_buffer(ptr, 1, 200)
- ptr.chars[1] = 'b'
- return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab"
-
+ ptr = lltype.malloc(STR, 3)
+ ptr.hash = 0x62
+ ptr.chars[0] = '0'
+ ptr.chars[1] = 'B'
+ ptr.chars[2] = 'C'
+ ptr2 = rgc.ll_shrink_array(ptr, 2)
+ return ((ptr == ptr2) +
+ ord(ptr2.chars[0]) +
+ (ord(ptr2.chars[1]) << 8) +
+ (len(ptr2.chars) << 16) +
+ (ptr2.hash << 24))
return f
- def test_resizable_buffer(self):
- run = self.runner("resizable_buffer")
- assert run([]) == 1
+ def test_shrink_array(self):
+ run = self.runner("shrink_array")
+ if self.GC_CAN_SHRINK_ARRAY:
+ expected = 0x62024231
+ else:
+ expected = 0x62024230
+ assert run([]) == expected
def define_string_builder_over_allocation(cls):
import gc
@@ -1117,6 +1126,7 @@
class TestSemiSpaceGC(GenericMovingGCTests):
gcname = "semispace"
+ GC_CAN_SHRINK_ARRAY = True
class gcpolicy(gc.FrameworkGcPolicy):
class transformerclass(framework.FrameworkGCTransformer):
@@ -1138,6 +1148,7 @@
class TestGenerationGC(GenericMovingGCTests):
gcname = "generation"
+ GC_CAN_SHRINK_ARRAY = True
class gcpolicy(gc.FrameworkGcPolicy):
class transformerclass(framework.FrameworkGCTransformer):
Modified: pypy/trunk/pypy/rpython/test/test_rbuilder.py
==============================================================================
--- pypy/trunk/pypy/rpython/test/test_rbuilder.py (original)
+++ pypy/trunk/pypy/rpython/test/test_rbuilder.py Tue Jan 26 17:44:40 2010
@@ -1,7 +1,25 @@
from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
+from pypy.rpython.lltypesystem.rbuilder import *
+from pypy.rpython.annlowlevel import llstr, hlstr
from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+
+class TestStringBuilderDirect(object):
+ def test_simple(self):
+ sb = StringBuilderRepr.ll_new(3)
+ StringBuilderRepr.ll_append_char(sb, 'x')
+ StringBuilderRepr.ll_append(sb, llstr("abc"))
+ StringBuilderRepr.ll_append_slice(sb, llstr("foobar"), 2, 5)
+ StringBuilderRepr.ll_append_multiple_char(sb, 'y', 3)
+ s = StringBuilderRepr.ll_build(sb)
+ assert hlstr(s) == "xabcobayyy"
+
+ def test_nooveralloc(self):
+ sb = StringBuilderRepr.ll_new(3)
+ StringBuilderRepr.ll_append(sb, llstr("abc"))
+ assert StringBuilderRepr.ll_build(sb) == sb.buf
+
class BaseTestStringBuilder(BaseRtypingTest):
def test_simple(self):
def func():
@@ -37,7 +55,6 @@
assert res == 'aabcabcdefbuuuu'
assert isinstance(res, unicode)
-
class TestLLtype(BaseTestStringBuilder, LLRtypeMixin):
pass
Modified: pypy/trunk/pypy/translator/c/src/mem.h
==============================================================================
--- pypy/trunk/pypy/translator/c/src/mem.h (original)
+++ pypy/trunk/pypy/translator/c/src/mem.h Tue Jan 26 17:44:40 2010
@@ -101,10 +101,6 @@
#define OP_RAW_MALLOC_USAGE(size, r) r = size
-#define OP_RAW_REALLOC_SHRINK(p, old_size, size, r) r = PyObject_Realloc((void*)p, size)
-
-#define OP_RAW_REALLOC_GROW(p, old_size, size, r) r = PyObject_Realloc((void*)p, size)
-
#ifdef MS_WINDOWS
#define alloca _alloca
#endif
Modified: pypy/trunk/pypy/translator/c/test/test_boehm.py
==============================================================================
--- pypy/trunk/pypy/translator/c/test/test_boehm.py (original)
+++ pypy/trunk/pypy/translator/c/test/test_boehm.py Tue Jan 26 17:44:40 2010
@@ -411,20 +411,25 @@
run = self.getcompiled(func)
assert run() == 0
- def test_resizable_buffer(self):
+ def test_shrink_array(self):
from pypy.rpython.lltypesystem.rstr import STR
- from pypy.rpython.annlowlevel import hlstr
from pypy.rlib import rgc
def f():
- ptr = rgc.resizable_buffer_of_shape(STR, 2)
- ptr.chars[0] = 'a'
- ptr = rgc.resize_buffer(ptr, 1, 200)
- ptr.chars[1] = 'b'
- return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab"
+ ptr = lltype.malloc(STR, 3)
+ ptr.hash = 0x62
+ ptr.chars[0] = '0'
+ ptr.chars[1] = 'B'
+ ptr.chars[2] = 'C'
+ ptr2 = rgc.ll_shrink_array(ptr, 2)
+ return ((ptr == ptr2) +
+ ord(ptr2.chars[0]) +
+ (ord(ptr2.chars[1]) << 8) +
+ (len(ptr2.chars) << 16) +
+ (ptr2.hash << 24))
run = self.getcompiled(f)
- assert run() == True
+ assert run() == 0x62024230
def test_assume_young_pointers_nop(self):
S = lltype.GcStruct('S', ('x', lltype.Signed))
Modified: pypy/trunk/pypy/translator/c/test/test_newgc.py
==============================================================================
--- pypy/trunk/pypy/translator/c/test/test_newgc.py (original)
+++ pypy/trunk/pypy/translator/c/test/test_newgc.py Tue Jan 26 17:44:40 2010
@@ -20,6 +20,7 @@
taggedpointers = False
GC_CAN_MOVE = False
GC_CANNOT_MALLOC_NONMOVABLE = False
+ GC_CAN_SHRINK_ARRAY = False
_isolated_func = None
@@ -697,19 +698,28 @@
def define_resizable_buffer(cls):
from pypy.rpython.lltypesystem.rstr import STR
- from pypy.rpython.annlowlevel import hlstr
def f():
- ptr = rgc.resizable_buffer_of_shape(STR, 2)
- ptr.chars[0] = 'a'
- ptr = rgc.resize_buffer(ptr, 1, 200)
- ptr.chars[1] = 'b'
- return hlstr(rgc.finish_building_buffer(ptr, 2)) == "ab"
-
+ ptr = lltype.malloc(STR, 3)
+ ptr.hash = 0x62
+ ptr.chars[0] = '0'
+ ptr.chars[1] = 'B'
+ ptr.chars[2] = 'C'
+ ptr2 = rgc.ll_shrink_array(ptr, 2)
+ return ((ptr == ptr2) +
+ ord(ptr2.chars[0]) +
+ (ord(ptr2.chars[1]) << 8) +
+ (len(ptr2.chars) << 16) +
+ (ptr2.hash << 24))
return f
def test_resizable_buffer(self):
- assert self.run('resizable_buffer')
+ res = self.run('resizable_buffer')
+ if self.GC_CAN_SHRINK_ARRAY:
+ expected = 0x62024231
+ else:
+ expected = 0x62024230
+ assert res == expected
def define_hash_preservation(cls):
from pypy.rlib.objectmodel import compute_hash
@@ -882,6 +892,7 @@
should_be_moving = True
GC_CAN_MOVE = True
GC_CANNOT_MALLOC_NONMOVABLE = True
+ GC_CAN_SHRINK_ARRAY = True
# for snippets
large_tests_ok = True
@@ -1029,6 +1040,7 @@
class TestMarkCompactGC(TestSemiSpaceGC):
gcpolicy = "markcompact"
should_be_moving = True
+ GC_CAN_SHRINK_ARRAY = False
def setup_class(cls):
py.test.skip("Disabled for now")
More information about the Pypy-commit
mailing list