[pypy-commit] pypy ootype-rerased: Merge default.
ademan
noreply at buildbot.pypy.org
Sun Jul 24 03:57:51 CEST 2011
Author: Daniel Roberts <Ademan555 at gmail.com>
Branch: ootype-rerased
Changeset: r45924:ffc2e849fd16
Date: 2011-07-23 18:57 -0700
http://bitbucket.org/pypy/pypy/changeset/ffc2e849fd16/
Log: Merge default.
diff --git a/pypy/config/support.py b/pypy/config/support.py
--- a/pypy/config/support.py
+++ b/pypy/config/support.py
@@ -9,7 +9,7 @@
return 1 # don't override MAKEFLAGS. This will call 'make' without any '-j' option
if sys.platform == 'darwin':
return darwin_get_cpu_count()
- elif sys.platform != 'linux2':
+ elif not sys.platform.startswith('linux'):
return 1 # implement me
try:
if isinstance(filename_or_file, str):
diff --git a/pypy/config/test/test_support.py b/pypy/config/test/test_support.py
--- a/pypy/config/test/test_support.py
+++ b/pypy/config/test/test_support.py
@@ -40,7 +40,7 @@
return self._value
def test_cpuinfo_linux():
- if sys.platform != 'linux2':
+ if not sys.platform.startswith('linux'):
py.test.skip("linux only")
saved = os.environ
try:
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -23,7 +23,7 @@
self.hasdict |= __base.hasdict
self.weakrefable |= __base.weakrefable
self.rawdict = {}
- self.acceptable_as_base_class = True
+ self.acceptable_as_base_class = '__new__' in rawdict
self.applevel_subclasses_base = None
# xxx used by faking
self.fakedcpytype = None
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -453,21 +453,33 @@
class WriteBarrierDescr(AbstractDescr):
def __init__(self, gc_ll_descr):
+ GCClass = gc_ll_descr.GCClass
self.llop1 = gc_ll_descr.llop1
self.WB_FUNCPTR = gc_ll_descr.WB_FUNCPTR
self.WB_ARRAY_FUNCPTR = gc_ll_descr.WB_ARRAY_FUNCPTR
- self.fielddescr_tid = get_field_descr(gc_ll_descr,
- gc_ll_descr.GCClass.HDR, 'tid')
- self.jit_wb_if_flag = gc_ll_descr.GCClass.JIT_WB_IF_FLAG
- # if convenient for the backend, we also compute the info about
+ self.fielddescr_tid = get_field_descr(gc_ll_descr, GCClass.HDR, 'tid')
+ #
+ self.jit_wb_if_flag = GCClass.JIT_WB_IF_FLAG
+ self.jit_wb_if_flag_byteofs, self.jit_wb_if_flag_singlebyte = (
+ self.extract_flag_byte(self.jit_wb_if_flag))
+ #
+ if hasattr(GCClass, 'JIT_WB_CARDS_SET'):
+ self.jit_wb_cards_set = GCClass.JIT_WB_CARDS_SET
+ self.jit_wb_card_page_shift = GCClass.JIT_WB_CARD_PAGE_SHIFT
+ self.jit_wb_cards_set_byteofs, self.jit_wb_cards_set_singlebyte = (
+ self.extract_flag_byte(self.jit_wb_cards_set))
+ else:
+ self.jit_wb_cards_set = 0
+
+ def extract_flag_byte(self, flag_word):
+ # if convenient for the backend, we compute the info about
# the flag as (byte-offset, single-byte-flag).
import struct
- value = struct.pack("l", self.jit_wb_if_flag)
+ value = struct.pack("l", flag_word)
assert value.count('\x00') == len(value) - 1 # only one byte is != 0
i = 0
while value[i] == '\x00': i += 1
- self.jit_wb_if_flag_byteofs = i
- self.jit_wb_if_flag_singlebyte = struct.unpack('b', value[i])[0]
+ return (i, struct.unpack('b', value[i])[0])
def get_write_barrier_fn(self, cpu):
llop1 = self.llop1
diff --git a/pypy/jit/backend/llvm/llvm_rffi.py b/pypy/jit/backend/llvm/llvm_rffi.py
--- a/pypy/jit/backend/llvm/llvm_rffi.py
+++ b/pypy/jit/backend/llvm/llvm_rffi.py
@@ -3,7 +3,7 @@
from pypy.rpython.lltypesystem import lltype, rffi
from pypy.translator.tool.cbuild import ExternalCompilationInfo, log
-if sys.platform != 'linux2':
+if not sys.platform.startswith('linux'):
py.test.skip("Linux only for now")
# ____________________________________________________________
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1707,6 +1707,7 @@
jit_wb_if_flag = 4096
jit_wb_if_flag_byteofs = struct.pack("i", 4096).index('\x10')
jit_wb_if_flag_singlebyte = 0x10
+ jit_wb_cards_set = 0
def get_write_barrier_from_array_fn(self, cpu):
return funcbox.getint()
#
@@ -1728,6 +1729,72 @@
else:
assert record == []
+ def test_cond_call_gc_wb_array_card_marking_fast_path(self):
+ def func_void(a, b, c):
+ record.append((a, b, c))
+ record = []
+ #
+ S = lltype.Struct('S', ('tid', lltype.Signed))
+ S_WITH_CARDS = lltype.Struct('S_WITH_CARDS',
+ ('card0', lltype.Char),
+ ('card1', lltype.Char),
+ ('card2', lltype.Char),
+ ('card3', lltype.Char),
+ ('card4', lltype.Char),
+ ('card5', lltype.Char),
+ ('card6', lltype.Char),
+ ('card7', lltype.Char),
+ ('data', S))
+ FUNC = self.FuncType([lltype.Ptr(S), lltype.Signed, lltype.Ptr(S)],
+ lltype.Void)
+ func_ptr = llhelper(lltype.Ptr(FUNC), func_void)
+ funcbox = self.get_funcbox(self.cpu, func_ptr)
+ class WriteBarrierDescr(AbstractDescr):
+ jit_wb_if_flag = 4096
+ jit_wb_if_flag_byteofs = struct.pack("i", 4096).index('\x10')
+ jit_wb_if_flag_singlebyte = 0x10
+ jit_wb_cards_set = 8192
+ jit_wb_cards_set_byteofs = struct.pack("i", 8192).index('\x20')
+ jit_wb_cards_set_singlebyte = 0x20
+ jit_wb_card_page_shift = 7
+ def get_write_barrier_from_array_fn(self, cpu):
+ return funcbox.getint()
+ #
+ for BoxIndexCls in [BoxInt, ConstInt]:
+ for cond in [False, True]:
+ print
+ print '_'*79
+ print 'BoxIndexCls =', BoxIndexCls
+ print 'JIT_WB_CARDS_SET =', cond
+ print
+ value = random.randrange(-sys.maxint, sys.maxint)
+ value |= 4096
+ if cond:
+ value |= 8192
+ else:
+ value &= ~8192
+ s = lltype.malloc(S_WITH_CARDS, immortal=True, zero=True)
+ s.data.tid = value
+ sgcref = rffi.cast(llmemory.GCREF, s.data)
+ del record[:]
+ box_index = BoxIndexCls((9<<7) + 17)
+ self.execute_operation(rop.COND_CALL_GC_WB_ARRAY,
+ [BoxPtr(sgcref), box_index, BoxPtr(sgcref)],
+ 'void', descr=WriteBarrierDescr())
+ if cond:
+ assert record == []
+ assert s.card6 == '\x02'
+ else:
+ assert record == [(s.data, (9<<7) + 17, s.data)]
+ assert s.card6 == '\x00'
+ assert s.card0 == '\x00'
+ assert s.card1 == '\x00'
+ assert s.card2 == '\x00'
+ assert s.card3 == '\x00'
+ assert s.card4 == '\x00'
+ assert s.card5 == '\x00'
+ assert s.card7 == '\x00'
+
def test_force_operations_returning_void(self):
values = []
def maybe_force(token, flag):
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -2246,10 +2246,12 @@
if opnum == rop.COND_CALL_GC_WB:
N = 2
func = descr.get_write_barrier_fn(self.cpu)
+ card_marking = False
elif opnum == rop.COND_CALL_GC_WB_ARRAY:
N = 3
func = descr.get_write_barrier_from_array_fn(self.cpu)
assert func != 0
+ card_marking = descr.jit_wb_cards_set != 0
else:
raise AssertionError(opnum)
#
@@ -2258,6 +2260,18 @@
imm(descr.jit_wb_if_flag_singlebyte))
self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later
jz_location = self.mc.get_relative_pos()
+
+ # for cond_call_gc_wb_array, also add another fast path:
+ # if GCFLAG_CARDS_SET, then we can just set one bit and be done
+ if card_marking:
+ self.mc.TEST8(addr_add_const(loc_base,
+ descr.jit_wb_cards_set_byteofs),
+ imm(descr.jit_wb_cards_set_singlebyte))
+ self.mc.J_il8(rx86.Conditions['NZ'], 0) # patched later
+ jnz_location = self.mc.get_relative_pos()
+ else:
+ jnz_location = 0
+
# the following is supposed to be the slow path, so whenever possible
# we choose the most compact encoding over the most efficient one.
if IS_X86_32:
@@ -2297,6 +2311,43 @@
loc = arglocs[i]
assert isinstance(loc, RegLoc)
self.mc.POP_r(loc.value)
+
+ # if GCFLAG_CARDS_SET, then we can do the whole thing that would
+ # be done in the CALL above with just four instructions, so here
+ # is an inline copy of them
+ if card_marking:
+ self.mc.JMP_l8(0) # jump to the exit, patched later
+ jmp_location = self.mc.get_relative_pos()
+ # patch the JNZ above
+ offset = self.mc.get_relative_pos() - jnz_location
+ assert 0 < offset <= 127
+ self.mc.overwrite(jnz_location-1, chr(offset))
+ #
+ loc_index = arglocs[1]
+ if isinstance(loc_index, RegLoc):
+ # choose a scratch register
+ tmp1 = loc_index
+ self.mc.PUSH_r(tmp1.value)
+ # SHR tmp, card_page_shift
+ self.mc.SHR_ri(tmp1.value, descr.jit_wb_card_page_shift)
+ # XOR tmp, -8
+ self.mc.XOR_ri(tmp1.value, -8)
+ # BTS [loc_base], tmp
+ self.mc.BTS(addr_add_const(loc_base, 0), tmp1)
+ # done
+ self.mc.POP_r(tmp1.value)
+ elif isinstance(loc_index, ImmedLoc):
+ byte_index = loc_index.value >> descr.jit_wb_card_page_shift
+ byte_ofs = ~(byte_index >> 3)
+ byte_val = 1 << (byte_index & 7)
+ self.mc.OR8(addr_add_const(loc_base, byte_ofs), imm(byte_val))
+ else:
+ raise AssertionError("index is neither RegLoc nor ImmedLoc")
+ # patch the JMP above
+ offset = self.mc.get_relative_pos() - jmp_location
+ assert 0 < offset <= 127
+ self.mc.overwrite(jmp_location-1, chr(offset))
+ #
# patch the JZ above
offset = self.mc.get_relative_pos() - jz_location
assert 0 < offset <= 127
diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py
--- a/pypy/jit/backend/x86/regloc.py
+++ b/pypy/jit/backend/x86/regloc.py
@@ -476,6 +476,7 @@
AND = _binaryop('AND')
OR = _binaryop('OR')
+ OR8 = _binaryop('OR8')
XOR = _binaryop('XOR')
NOT = _unaryop('NOT')
SHL = _binaryop('SHL')
@@ -483,6 +484,7 @@
SAR = _binaryop('SAR')
TEST = _binaryop('TEST')
TEST8 = _binaryop('TEST8')
+ BTS = _binaryop('BTS')
ADD = _binaryop('ADD')
SUB = _binaryop('SUB')
diff --git a/pypy/jit/backend/x86/rx86.py b/pypy/jit/backend/x86/rx86.py
--- a/pypy/jit/backend/x86/rx86.py
+++ b/pypy/jit/backend/x86/rx86.py
@@ -496,6 +496,10 @@
AND8_rr = insn(rex_fw, '\x20', byte_register(1), byte_register(2,8), '\xC0')
OR8_rr = insn(rex_fw, '\x08', byte_register(1), byte_register(2,8), '\xC0')
+ OR8_mi = insn(rex_fw, '\x80', orbyte(1<<3), mem_reg_plus_const(1),
+ immediate(2, 'b'))
+ OR8_ji = insn(rex_fw, '\x80', orbyte(1<<3), abs_, immediate(1),
+ immediate(2, 'b'))
NEG_r = insn(rex_w, '\xF7', register(1), '\xD8')
@@ -565,6 +569,9 @@
TEST8_ji = insn(rex_nw, '\xF6', orbyte(0<<3), abs_, immediate(1), immediate(2, 'b'))
TEST_rr = insn(rex_w, '\x85', register(2,8), register(1), '\xC0')
+ BTS_mr = insn(rex_w, '\x0F\xAB', register(2,8), mem_reg_plus_const(1))
+ BTS_jr = insn(rex_w, '\x0F\xAB', register(2,8), abs_, immediate(1))
+
# x87 instructions
FSTP_b = insn('\xDD', orbyte(3<<3), stack_bp(1))
diff --git a/pypy/jit/backend/x86/test/test_zrpy_gc.py b/pypy/jit/backend/x86/test/test_zrpy_gc.py
--- a/pypy/jit/backend/x86/test/test_zrpy_gc.py
+++ b/pypy/jit/backend/x86/test/test_zrpy_gc.py
@@ -524,6 +524,76 @@
def test_compile_framework_8(self):
self.run('compile_framework_8')
+ def define_compile_framework_9(cls):
+ # Like compile_framework_8, but with variable indexes and large
+ # arrays, testing the card_marking case
+ def before(n, x):
+ return n, x, None, None, None, None, None, None, None, None, [X(123)], None
+ def f(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s):
+ if n < 1900:
+ check(l[0].x == 123)
+ num = 512 + (n & 7)
+ l = [None] * num
+ l[0] = X(123)
+ l[1] = X(n)
+ l[2] = X(n+10)
+ l[3] = X(n+20)
+ l[4] = X(n+30)
+ l[5] = X(n+40)
+ l[6] = X(n+50)
+ l[7] = X(n+60)
+ l[num-8] = X(n+70)
+ l[num-9] = X(n+80)
+ l[num-10] = X(n+90)
+ l[num-11] = X(n+100)
+ l[-12] = X(n+110)
+ l[-13] = X(n+120)
+ l[-14] = X(n+130)
+ l[-15] = X(n+140)
+ if n < 1800:
+ num = 512 + (n & 7)
+ check(len(l) == num)
+ check(l[0].x == 123)
+ check(l[1].x == n)
+ check(l[2].x == n+10)
+ check(l[3].x == n+20)
+ check(l[4].x == n+30)
+ check(l[5].x == n+40)
+ check(l[6].x == n+50)
+ check(l[7].x == n+60)
+ check(l[num-8].x == n+70)
+ check(l[num-9].x == n+80)
+ check(l[num-10].x == n+90)
+ check(l[num-11].x == n+100)
+ check(l[-12].x == n+110)
+ check(l[-13].x == n+120)
+ check(l[-14].x == n+130)
+ check(l[-15].x == n+140)
+ n -= x.foo
+ return n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s
+ def after(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s):
+ check(len(l) >= 512)
+ check(l[0].x == 123)
+ check(l[1].x == 2)
+ check(l[2].x == 12)
+ check(l[3].x == 22)
+ check(l[4].x == 32)
+ check(l[5].x == 42)
+ check(l[6].x == 52)
+ check(l[7].x == 62)
+ check(l[-8].x == 72)
+ check(l[-9].x == 82)
+ check(l[-10].x == 92)
+ check(l[-11].x == 102)
+ check(l[-12].x == 112)
+ check(l[-13].x == 122)
+ check(l[-14].x == 132)
+ check(l[-15].x == 142)
+ return before, f, after
+
+ def test_compile_framework_9(self):
+ self.run('compile_framework_9')
+
def define_compile_framework_external_exception_handling(cls):
def before(n, x):
x = X(0)
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -10,6 +10,7 @@
from pypy.rlib.objectmodel import we_are_translated
from pypy.rlib.unroll import unrolling_iterable
from pypy.rlib.debug import fatalerror
+from pypy.rlib.rstackovf import StackOverflow
from pypy.translator.simplify import get_functype
from pypy.translator.unsimplify import call_final_function
@@ -408,6 +409,15 @@
jd.warmstate = state
def crash_in_jit(e):
+ try:
+ raise e
+ except JitException:
+ raise # go through
+ except MemoryError:
+ raise # go through
+ except StackOverflow:
+ raise # go through
+ except Exception, e:
if not we_are_translated():
print "~~~ Crash in JIT!"
print '~~~ %s: %s' % (e.__class__, e)
@@ -421,8 +431,6 @@
def maybe_enter_jit(*args):
try:
maybe_compile_and_run(state.increment_threshold, *args)
- except JitException:
- raise # go through
except Exception, e:
crash_in_jit(e)
maybe_enter_jit._always_inline_ = True
diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py
--- a/pypy/module/__builtin__/abstractinst.py
+++ b/pypy/module/__builtin__/abstractinst.py
@@ -58,7 +58,10 @@
# -- case (anything, type)
try:
- w_result = space.isinstance(w_obj, w_klass_or_tuple, allow_override)
+ if allow_override:
+ w_result = space.isinstance_allow_override(w_obj, w_klass_or_tuple)
+ else:
+ w_result = space.isinstance(w_obj, w_klass_or_tuple)
except OperationError, e: # if w_klass_or_tuple was not a type, ignore it
if not e.match(space, space.w_TypeError):
raise # propagate other errors
@@ -72,8 +75,11 @@
w_pretendtype = space.getattr(w_obj, space.wrap('__class__'))
if space.is_w(w_pretendtype, space.type(w_obj)):
return False # common case: obj.__class__ is type(obj)
- w_result = space.issubtype(w_pretendtype, w_klass_or_tuple,
- allow_override)
+ if allow_override:
+ w_result = space.issubtype_allow_override(w_pretendtype,
+ w_klass_or_tuple)
+ else:
+ w_result = space.issubtype(w_pretendtype, w_klass_or_tuple)
except OperationError, e:
if e.async(space):
raise
@@ -134,7 +140,11 @@
# -- case (type, type)
try:
- w_result = space.issubtype(w_derived, w_klass_or_tuple, allow_override)
+ if allow_override:
+ w_result = space.issubtype_allow_override(w_derived,
+ w_klass_or_tuple)
+ else:
+ w_result = space.issubtype(w_derived, w_klass_or_tuple)
except OperationError, e: # if one of the args was not a type, ignore it
if not e.match(space, space.w_TypeError):
raise # propagate other errors
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -852,7 +852,7 @@
# Sometimes the library is wrapped into another DLL, ensure that
# the correct bootstrap code is installed
kwds["link_extra"] = ["msvcrt.lib"]
- elif sys.platform == 'linux2':
+ elif sys.platform.startswith('linux'):
compile_extra.append("-Werror=implicit-function-declaration")
export_symbols_eci.append('pypyAPI')
else:
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -188,7 +188,7 @@
kwds["compile_extra"] = ["/we4013"]
else:
kwds["link_files"] = [str(api_library + '.so')]
- if sys.platform == 'linux2':
+ if sys.platform.startswith('linux'):
kwds["compile_extra"]=["-Werror=implicit-function-declaration"]
return compile_module(self.space, name, **kwds)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -85,7 +85,7 @@
return SEARCH_ERROR, None, None
-if sys.platform == 'linux2' or 'freebsd' in sys.platform:
+if sys.platform.startswith('linux') or 'freebsd' in sys.platform:
def case_ok(filename):
return True
else:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -125,13 +125,13 @@
assert st.st_size == 14
assert st.st_nlink == 1
- #if sys.platform.startswith('linux2'):
+ #if sys.platform.startswith('linux'):
# # expects non-integer timestamps - it's unlikely that they are
# # all three integers
# assert ((st.st_atime, st.st_mtime, st.st_ctime) !=
# (st[7], st[8], st[9]))
# assert st.st_blksize * st.st_blocks >= st.st_size
- if sys.platform.startswith('linux2'):
+ if sys.platform.startswith('linux'):
assert hasattr(st, 'st_rdev')
def test_stat_float_times(self):
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -497,22 +497,25 @@
space.wrap("coercion should return None or 2-tuple"))
return w_res
- def issubtype(space, w_sub, w_type, allow_override=False):
- if allow_override:
+ def issubtype(space, w_sub, w_type):
+ return space._type_issubtype(w_sub, w_type)
+
+ def isinstance(space, w_inst, w_type):
+ return space._type_isinstance(w_inst, w_type)
+
+ def issubtype_allow_override(space, w_sub, w_type):
w_check = space.lookup(w_type, "__subclasscheck__")
if w_check is None:
raise OperationError(space.w_TypeError,
space.wrap("issubclass not supported here"))
return space.get_and_call_function(w_check, w_type, w_sub)
- return space._type_issubtype(w_sub, w_type)
- def isinstance(space, w_inst, w_type, allow_override=False):
- if allow_override:
+ def isinstance_allow_override(space, w_inst, w_type):
w_check = space.lookup(w_type, "__instancecheck__")
if w_check is not None:
return space.get_and_call_function(w_check, w_type, w_inst)
- return space.issubtype(space.type(w_inst), w_type, allow_override)
-
+ else:
+ return space.isinstance(w_inst, w_type)
# helpers
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -568,3 +568,8 @@
if isinstance(w_sub, W_TypeObject) and isinstance(w_type, W_TypeObject):
return self.wrap(w_sub.issubtype(w_type))
raise OperationError(self.w_TypeError, self.wrap("need type objects"))
+
+ def _type_isinstance(self, w_inst, w_type):
+ if isinstance(w_type, W_TypeObject):
+ return self.wrap(self.type(w_inst).issubtype(w_type))
+ raise OperationError(self.w_TypeError, self.wrap("need type object"))
diff --git a/pypy/rlib/test/test_rlocale.py b/pypy/rlib/test/test_rlocale.py
--- a/pypy/rlib/test/test_rlocale.py
+++ b/pypy/rlib/test/test_rlocale.py
@@ -37,7 +37,7 @@
assert isinstance(grouping, str)
def test_libintl():
- if sys.platform not in ("linux2", "darwin"):
+ if sys.platform != "darwin" or not sys.platform.startswith("linux"):
py.test.skip("there is (maybe) no libintl here")
_gettext = external('gettext', [rffi.CCHARP], rffi.CCHARP)
res = _gettext("1234")
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1331,7 +1331,7 @@
def _where_is_errno():
return standard_c_lib._errno()
- elif sys.platform in ('linux2', 'freebsd6'):
+ elif sys.platform.startswith('linux') or sys.platform == 'freebsd6':
standard_c_lib.__errno_location.restype = ctypes.POINTER(ctypes.c_int)
def _where_is_errno():
return standard_c_lib.__errno_location()
diff --git a/pypy/rpython/lltypesystem/llarena.py b/pypy/rpython/lltypesystem/llarena.py
--- a/pypy/rpython/lltypesystem/llarena.py
+++ b/pypy/rpython/lltypesystem/llarena.py
@@ -404,7 +404,7 @@
from pypy.rpython.extfunc import register_external
from pypy.rlib.objectmodel import CDefinedIntSymbolic
-if sys.platform == 'linux2':
+if sys.platform.startswith('linux'):
# This only works with linux's madvise(), which is really not a memory
# usage hint but a real command. It guarantees that after MADV_DONTNEED
# the pages are cleared again.
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1348,7 +1348,7 @@
def test_prefix(self):
- if sys.platform != 'linux2':
+ if not sys.platform.startswith('linux'):
py.test.skip("Not supported")
from pypy.translator.platform import platform
diff --git a/pypy/rpython/memory/gc/env.py b/pypy/rpython/memory/gc/env.py
--- a/pypy/rpython/memory/gc/env.py
+++ b/pypy/rpython/memory/gc/env.py
@@ -55,7 +55,7 @@
# will be huge on 64-bit systems.
if sys.maxint == 2147483647: # 32-bit
- if sys.platform == 'linux2':
+ if sys.platform.startswith('linux'):
addressable_size = float(2**32) # 4GB
elif sys.platform == 'win32':
addressable_size = float(2**31) # 2GB
@@ -65,7 +65,7 @@
addressable_size = float(2**63) # 64-bit
-def get_total_memory_linux2(filename):
+def get_total_memory_linux(filename):
debug_start("gc-hardware")
result = -1.0
try:
@@ -93,7 +93,7 @@
result = addressable_size
debug_stop("gc-hardware")
return result
-
+get_total_memory_linux2 = get_total_memory_linux3 = get_total_memory_linux
def get_total_memory_darwin(result):
debug_start("gc-hardware")
@@ -108,7 +108,7 @@
return result
-if sys.platform == 'linux2':
+if sys.platform.startswith('linux'):
def get_total_memory():
return get_total_memory_linux2('/proc/meminfo')
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -34,13 +34,6 @@
the GC in very small programs. Defaults to 8
times the nursery.
- PYPY_GC_LOSTCARD If between two minor collections we see more than
- 'PYPY_GC_LOSTCARD * length' writes to the same array,
- then give up card marking and use the fast write
- barrier instead. Defaults to 0.3333 for now.
- Avoid values lower than 0.125: it is the growth
- factor of list.append().
-
PYPY_GC_DEBUG Enable extra checks around collections that are
too slow for normal use. Values are 0 (off),
1 (on major collections) or 2 (also on minor
@@ -205,9 +198,6 @@
# larger. A value of 0 disables card marking.
"card_page_indices": 128,
- # See PYPY_GC_LOSTCARD.
- "lost_card": 1.0 / 3.0,
-
# Objects whose total size is at least 'large_object' bytes are
# allocated out of the nursery immediately, as old objects. The
# minimal allocated size of the nursery is 2x the following
@@ -224,7 +214,6 @@
major_collection_threshold=2.5,
growth_rate_max=2.5, # for tests
card_page_indices=0,
- lost_card=0.5,
large_object=8*WORD,
ArenaCollectionClass=None,
**kwds):
@@ -246,7 +235,6 @@
self.card_page_shift = 0
while (1 << self.card_page_shift) < self.card_page_indices:
self.card_page_shift += 1
- self.lost_card = lost_card
#
# 'large_object' limit how big objects can be in the nursery, so
# it gives a lower bound on the allowed size of the nursery.
@@ -275,8 +263,8 @@
# that it is possible for an object to be listed both in here
# and in 'objects_pointing_to_young', in which case we
# should just clear the cards and trace it fully, as usual.
- # Note also that young array objects may be added to this list.
- self.objects_with_cards_set = self.AddressStack()
+ # Note also that young array objects are never listed here.
+ self.old_objects_with_cards_set = self.AddressStack()
#
# A list of all prebuilt GC objects that contain pointers to the heap
self.prebuilt_root_objects = self.AddressStack()
@@ -367,10 +355,6 @@
else:
self.max_delta = 0.125 * env.get_total_memory()
#
- lost_card = env.read_float_from_env('PYPY_GC_LOSTCARD')
- if lost_card > 0.0:
- self.lost_card = lost_card
- #
self.minor_collection() # to empty the nursery
llarena.arena_free(self.nursery)
self.nursery_size = newsize
@@ -665,11 +649,15 @@
#
else:
# Reserve N extra words containing card bits before the object.
- extra_words = self.card_marking_words_for_length(length) + 1
+ extra_words = self.card_marking_words_for_length(length)
cardheadersize = WORD * extra_words
extra_flags = GCFLAG_HAS_CARDS | GCFLAG_TRACK_YOUNG_PTRS
- # note that if 'can_make_young', then card marking will only
- # be used later, after (and if) the object becomes old
+ # if 'can_make_young', then we also immediately set
+ # GCFLAG_CARDS_SET, but without adding the object to
+ # 'old_objects_with_cards_set'. In this way it should
+ # never be added to that list as long as it is young.
+ if can_make_young:
+ extra_flags |= GCFLAG_CARDS_SET
#
# Detect very rare cases of overflows
if raw_malloc_usage(totalsize) > (sys.maxint - (WORD-1)
@@ -691,15 +679,11 @@
raise MemoryError("cannot allocate large object")
#
# Reserve the card mark bits as a list of single bytes
- # followed by a Signed (the loop is empty in C).
- if cardheadersize > 0:
+ # (the loop is empty in C).
i = 0
- while i < cardheadersize - WORD:
- llarena.arena_reserve(arena + i,
- llmemory.sizeof(lltype.Char))
+ while i < cardheadersize:
+ llarena.arena_reserve(arena + i, llmemory.sizeof(lltype.Char))
i += 1
- llarena.arena_reserve(arena + i,
- llmemory.sizeof(lltype.Signed))
#
# Reserve the actual object. (This is also a no-op in C).
result = arena + cardheadersize
@@ -923,11 +907,14 @@
length = (obj + offset_to_length).signed[0]
extra_words = self.card_marking_words_for_length(length)
#
+ size_gc_header = self.gcheaderbuilder.size_gc_header
+ p = llarena.getfakearenaaddress(obj - size_gc_header)
i = extra_words * WORD
while i > 0:
+ p -= 1
+ ll_assert(p.char[0] == '\x00',
+ "the card marker bits are not cleared")
i -= 1
- ll_assert(self.get_card(obj, i).char[0] == '\x00',
- "the card marker bits are not cleared")
# ----------
# Write barrier
@@ -937,6 +924,20 @@
# "if addr_struct.int0 & JIT_WB_IF_FLAG: remember_young_pointer()")
JIT_WB_IF_FLAG = GCFLAG_TRACK_YOUNG_PTRS
+ # for the JIT to generate custom code corresponding to the array
+ # write barrier for the simplest case of cards. If JIT_CARDS_SET
+ # is already set on an object, it will execute code like this:
+ # MOV eax, index
+ # SHR eax, JIT_WB_CARD_PAGE_SHIFT
+ # XOR eax, -8
+ # BTS [object], eax
+ if TRANSLATION_PARAMS['card_page_indices'] > 0:
+ JIT_WB_CARDS_SET = GCFLAG_CARDS_SET
+ JIT_WB_CARD_PAGE_SHIFT = 1
+ while ((1 << JIT_WB_CARD_PAGE_SHIFT) !=
+ TRANSLATION_PARAMS['card_page_indices']):
+ JIT_WB_CARD_PAGE_SHIFT += 1
+
@classmethod
def JIT_max_size_of_young_obj(cls):
return cls.TRANSLATION_PARAMS['large_object']
@@ -1025,8 +1026,6 @@
self.prebuilt_root_objects.append(addr_array)
return
#
- self.set_cards_flag(addr_array)
- #
# 'addr_array' is a raw_malloc'ed array with card markers
# in front. Compute the index of the bit to set:
bitindex = index >> self.card_page_shift
@@ -1044,6 +1043,10 @@
# it seems more important that remember_young_pointer_from_array2()
# does not take 3 arguments).
addr_byte.char[0] = chr(byte | bitmask)
+ #
+ if objhdr.tid & GCFLAG_CARDS_SET == 0:
+ self.old_objects_with_cards_set.append(addr_array)
+ objhdr.tid |= GCFLAG_CARDS_SET
remember_young_pointer_from_array2._dont_inline_ = True
assert self.card_page_indices > 0
@@ -1072,8 +1075,6 @@
if not self.appears_to_be_young(newvalue):
return
#
- self.set_cards_flag(addr_array)
- #
# 'addr_array' is a raw_malloc'ed array with card markers
# in front. Compute the index of the bit to set:
bitindex = index >> self.card_page_shift
@@ -1086,6 +1087,10 @@
if byte & bitmask:
return
addr_byte.char[0] = chr(byte | bitmask)
+ #
+ if objhdr.tid & GCFLAG_CARDS_SET == 0:
+ self.old_objects_with_cards_set.append(addr_array)
+ objhdr.tid |= GCFLAG_CARDS_SET
return
#
# Logic for the no-cards case, put here to minimize the number
@@ -1103,36 +1108,11 @@
self.remember_young_pointer_from_array3 = (
remember_young_pointer_from_array3)
- def get_card_counter_addr(self, obj):
+ def get_card(self, obj, byteindex):
size_gc_header = self.gcheaderbuilder.size_gc_header
addr_byte = obj - size_gc_header
- return llarena.getfakearenaaddress(addr_byte) - WORD
+ return llarena.getfakearenaaddress(addr_byte) + (~byteindex)
- def get_card(self, obj, byteindex):
- return self.get_card_counter_addr(obj) + (~byteindex)
-
- def set_cards_flag(self, obj):
- hdr = self.header(obj)
- if hdr.tid & GCFLAG_CARDS_SET == 0:
- #
- # first time we set a card bit in this object
- self.header(obj).tid |= GCFLAG_CARDS_SET
- self.objects_with_cards_set.append(obj)
- #
- # initialize the counter with the array length and self.lost_card
- typeid = self.get_type_id(obj)
- offset_to_length = self.varsize_offset_to_length(typeid)
- length = (obj + offset_to_length).signed[0]
- counter = int(length * self.lost_card)
- self.get_card_counter_addr(obj).signed[0] = counter
- else:
- # decrement the counter and if zero is reached, give up on
- # card marking (up to the next collection).
- addr = self.get_card_counter_addr(obj)
- addr.signed[0] -= 1
- if addr.signed[0] < 0:
- self.objects_pointing_to_young.append(obj)
- hdr.tid &= ~GCFLAG_TRACK_YOUNG_PTRS
def assume_young_pointers(self, addr_struct):
"""Called occasionally by the JIT to mean ``assume that 'addr_struct'
@@ -1197,15 +1177,21 @@
# manually copy the individual card marks from source to dest
bytes = self.card_marking_bytes_for_length(length)
#
+ anybyte = 0
i = 0
while i < bytes:
addr_srcbyte = self.get_card(source_addr, i)
addr_dstbyte = self.get_card(dest_addr, i)
byte = ord(addr_srcbyte.char[0])
+ anybyte |= byte
addr_dstbyte.char[0] = chr(ord(addr_dstbyte.char[0]) | byte)
i += 1
#
- self.set_cards_flag(dest_addr)
+ if anybyte:
+ dest_hdr = self.header(dest_addr)
+ if dest_hdr.tid & GCFLAG_CARDS_SET == 0:
+ self.old_objects_with_cards_set.append(dest_addr)
+ dest_hdr.tid |= GCFLAG_CARDS_SET
# ----------
# Nursery collection
@@ -1239,9 +1225,9 @@
self.collect_oldrefs_to_nursery()
#
# We have to loop back if collect_oldrefs_to_nursery caused
- # new objects to show up in objects_with_cards_set
+ # new objects to show up in old_objects_with_cards_set
if self.card_page_indices > 0:
- if self.objects_with_cards_set.non_empty():
+ if self.old_objects_with_cards_set.non_empty():
continue
break
#
@@ -1284,7 +1270,7 @@
def collect_cardrefs_to_nursery(self):
size_gc_header = self.gcheaderbuilder.size_gc_header
- oldlist = self.objects_with_cards_set
+ oldlist = self.old_objects_with_cards_set
while oldlist.non_empty():
obj = oldlist.pop()
#
@@ -1299,7 +1285,6 @@
length = (obj + offset_to_length).signed[0]
bytes = self.card_marking_bytes_for_length(length)
p = llarena.getfakearenaaddress(obj - size_gc_header)
- p -= WORD
#
# If the object doesn't have GCFLAG_TRACK_YOUNG_PTRS, then it
# means that it is in 'objects_pointing_to_young' and
@@ -1400,22 +1385,7 @@
# arrive here.
if (bool(self.young_rawmalloced_objects)
and self.young_rawmalloced_objects.contains(obj)):
- # 'obj' points to a young, raw-malloced object
- if (self.header(obj).tid & GCFLAG_VISITED) == 0:
- self.header(obj).tid |= GCFLAG_VISITED
- #
- # we just made 'obj' old, so we may need to add it
- # in the correct list:
- if self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
- # common case: GCFLAG_TRACK_YOUNG_PTRS is not set, so
- # the object may contain young pointers anywhere
- self.objects_pointing_to_young.append(obj)
- else:
- # large array case: the object contains card marks
- # that tell us where young pointers are, and it
- # is already in objects_with_cards_set.
- ll_assert(self.header(obj).tid & GCFLAG_HAS_CARDS != 0,
- "neither YOUNG_PTRS nor HAS_CARDS??")
+ self._visit_young_rawmalloced_object(obj)
return
#
# If 'obj' was already forwarded, change it to its forwarding address.
@@ -1468,6 +1438,47 @@
# objects when we walk 'objects_pointing_to_young'.
self.objects_pointing_to_young.append(newobj)
+ def _visit_young_rawmalloced_object(self, obj):
+ # 'obj' points to a young, raw-malloced object.
+ # Any young rawmalloced object never seen by the code here
+ # will end up without GCFLAG_VISITED, and be freed at the
+ # end of the current minor collection. Note that there was
+ # a bug in which dying young arrays with card marks would
+ # still be scanned before being freed, keeping a lot of
+ # objects unnecessarily alive.
+ hdr = self.header(obj)
+ if hdr.tid & GCFLAG_VISITED:
+ return
+ hdr.tid |= GCFLAG_VISITED
+ #
+ # we just made 'obj' old, so we need to add it to the correct
+ # lists. (Note that another point of view on the longish
+ # comments below is that we are not changing any flags in 'hdr',
+ # but just restoring invariants: the object may be missing from
+ # these lists as long as it is a young array, but not when it
+ # grows old.)
+ anywhere = False
+ #
+ if hdr.tid & GCFLAG_TRACK_YOUNG_PTRS == 0:
+ # common case: GCFLAG_TRACK_YOUNG_PTRS is not set, so
+ # the object may contain young pointers anywhere
+ self.objects_pointing_to_young.append(obj)
+ anywhere = True
+ #
+ if hdr.tid & GCFLAG_HAS_CARDS != 0:
+ # large array case: the object contains card marks
+ # that tell us where young pointers are, and it must
+ # be added to 'old_objects_with_cards_set'. Note that
+ # we must add it even if we also added it just above to
+ # 'objects_pointing_to_young', because the object header
+ # needs to be cleaned up.
+ ll_assert(hdr.tid & GCFLAG_CARDS_SET != 0,
+ "young array: GCFLAG_HAS_CARDS without GCFLAG_CARDS_SET")
+ self.old_objects_with_cards_set.append(obj)
+ anywhere = True
+ #
+ ll_assert(anywhere, "wrong flag combination on young array")
+
def _malloc_out_of_nursery(self, totalsize):
"""Allocate non-movable memory for an object of the given
@@ -1638,7 +1649,7 @@
"GCFLAG_HAS_CARDS but not has_gcptr_in_varsize")
offset_to_length = self.varsize_offset_to_length(typeid)
length = (obj + offset_to_length).signed[0]
- extra_words = self.card_marking_words_for_length(length) + 1
+ extra_words = self.card_marking_words_for_length(length)
arena -= extra_words * WORD
allocsize += extra_words * WORD
#
diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -21,7 +21,7 @@
# sub-second timestamps.
# - TIMESPEC is defined when the "struct stat" contains st_atim field.
-if sys.platform == 'linux2':
+if sys.platform.startswith('linux'):
TIMESPEC = platform.Struct('struct timespec',
[('tv_sec', rffi.TIME_T),
('tv_nsec', rffi.LONG)])
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -226,7 +226,7 @@
eci = eci.merge(configure_boehm())
pre_include_bits = []
- if sys.platform == "linux2":
+ if sys.platform.startswith('linux'):
pre_include_bits += ["#define _REENTRANT 1",
"#define GC_LINUX_THREADS 1"]
if sys.platform != "win32":
diff --git a/pypy/translator/platform/__init__.py b/pypy/translator/platform/__init__.py
--- a/pypy/translator/platform/__init__.py
+++ b/pypy/translator/platform/__init__.py
@@ -230,7 +230,7 @@
return True
-if sys.platform == 'linux2':
+if sys.platform.startswith('linux'):
from pypy.translator.platform.linux import Linux, Linux64
import platform
if platform.architecture()[0] == '32bit':
diff --git a/pypy/translator/sandbox/sandlib.py b/pypy/translator/sandbox/sandlib.py
--- a/pypy/translator/sandbox/sandlib.py
+++ b/pypy/translator/sandbox/sandlib.py
@@ -209,7 +209,7 @@
def handle_until_return(self):
child_stdin = self.popen.stdin
child_stdout = self.popen.stdout
- if self.os_level_sandboxing and sys.platform.startswith('linux2'):
+ if self.os_level_sandboxing and sys.platform.startswith('linux'):
# rationale: we wait until the child process started completely,
# letting the C library do any system calls it wants for
# initialization. When the RPython code starts up, it quickly
diff --git a/pypy/translator/sandbox/test/test_sandbox.py b/pypy/translator/sandbox/test/test_sandbox.py
--- a/pypy/translator/sandbox/test/test_sandbox.py
+++ b/pypy/translator/sandbox/test/test_sandbox.py
@@ -145,7 +145,7 @@
g = pipe.stdin
f = pipe.stdout
expect(f, g, "ll_os.ll_os_getenv", ("PYPY_GENERATIONGC_NURSERY",), None)
- if sys.platform == 'linux2': # on Mac, uses another (sandboxsafe) approach
+ if sys.platform.startswith('linux'): # on Mac, uses another (sandboxsafe) approach
expect(f, g, "ll_os.ll_os_open", ("/proc/cpuinfo", 0, 420),
OSError(5232, "xyz"))
expect(f, g, "ll_os.ll_os_getenv", ("PYPY_GC_DEBUG",), None)
More information about the pypy-commit
mailing list