From santagada at codespeak.net Wed Dec 1 05:45:19 2010 From: santagada at codespeak.net (santagada at codespeak.net) Date: Wed, 1 Dec 2010 05:45:19 +0100 (CET) Subject: [pypy-svn] r79702 - pypy/trunk/pypy/module/_minimal_curses Message-ID: <20101201044519.B4C9136C225@codespeak.net> Author: santagada Date: Wed Dec 1 05:45:15 2010 New Revision: 79702 Modified: pypy/trunk/pypy/module/_minimal_curses/__init__.py Log: raise and ImportError instead of py.test.skip so the error is clearer for python users. Modified: pypy/trunk/pypy/module/_minimal_curses/__init__.py ============================================================================== --- pypy/trunk/pypy/module/_minimal_curses/__init__.py (original) +++ pypy/trunk/pypy/module/_minimal_curses/__init__.py Wed Dec 1 05:45:15 2010 @@ -4,7 +4,7 @@ try: import _minimal_curses as _curses # when running on top of pypy-c except ImportError: - import py; py.test.skip("no _curses module") # no _curses at all + raise ImportError("no _curses or _minimal_curses module") # no _curses at all from pypy.interpreter.mixedmodule import MixedModule from pypy.module._minimal_curses import fficurses From afa at codespeak.net Wed Dec 1 11:09:50 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 1 Dec 2010 11:09:50 +0100 (CET) Subject: [pypy-svn] r79703 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20101201100950.6090A282B9D@codespeak.net> Author: afa Date: Wed Dec 1 11:09:47 2010 New Revision: 79703 Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Log: Don't cheat and assume that float.__getformat__ may be missing from the host python. Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Wed Dec 1 11:09:47 2010 @@ -59,8 +59,34 @@ return w_obj -_float_format = float.__getformat__("float") -_double_format = float.__getformat__("double") +def detect_floatformat(): + from pypy.rpython.lltypesystem import rffi, lltype + buf = lltype.malloc(rffi.CCHARP.TO, 8, flavor='raw') + rffi.cast(rffi.DOUBLEP, buf)[0] = 9006104071832581.0 + packed = rffi.charpsize2str(buf, 8) + if packed == "\x43\x3f\xff\x01\x02\x03\x04\x05": + double_format = 'IEEE, big-endian' + elif packed == "\x05\x04\x03\x02\x01\xff\x3f\x43": + double_format = 'IEEE, little-endian' + else: + double_format = 'unknown' + lltype.free(buf, flavor='raw') + # + buf = lltype.malloc(rffi.CCHARP.TO, 4, flavor='raw') + rffi.cast(rffi.FLOATP, buf)[0] = rarithmetic.r_singlefloat(16711938.0) + packed = rffi.charpsize2str(buf, 4) + if packed == "\x4b\x7f\x01\x02": + float_format = 'IEEE, big-endian' + elif packed == "\x02\x01\x7f\x4b": + float_format = 'IEEE, little-endian' + else: + float_format = 'unknown' + lltype.free(buf, flavor='raw') + + return double_format, float_format + +_double_format, _float_format = detect_floatformat() + def descr___getformat__(space, w_cls, kind): if kind == "float": return space.wrap(_float_format) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_floatobject.py Wed Dec 1 11:09:47 2010 @@ -361,8 +361,8 @@ assert not (nan > x) def test___getformat__(self): - float.__getformat__("float") - float.__getformat__("double") + assert float.__getformat__("float") != "unknown" + assert float.__getformat__("double") != "unknown" raises(ValueError, float.__getformat__, "random") def test_trunc(self): From arigo at codespeak.net Wed Dec 1 12:06:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 1 Dec 2010 12:06:59 +0100 (CET) Subject: [pypy-svn] r79705 - in pypy/trunk/pypy/jit/backend: llsupport llsupport/test x86 x86/test Message-ID: <20101201110659.A4E14282BDD@codespeak.net> Author: arigo Date: Wed Dec 1 12:06:57 2010 New Revision: 79705 Modified: pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py pypy/trunk/pypy/jit/backend/llsupport/gc.py pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/regloc.py pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Log: Merge branch/jit-free-asm2, finishing the freeing of memory from the x86 backend. It moves some of the allocation logic out of the x86 backend, too. Modified: pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py Wed Dec 1 12:06:57 2010 @@ -43,6 +43,22 @@ self.total_mallocs -= (stop - start) self._add_free_block(start, stop) + def open_malloc(self, minsize): + """Allocate at least minsize bytes. Returns (start, stop).""" + result = self._allocate_block(minsize) + (start, stop) = result + self.total_mallocs += stop - start + return result + + def open_free(self, middle, stop): + """Used for freeing the end of an open-allocated block of memory.""" + if stop - middle >= self.min_fragment: + self.total_mallocs -= (stop - middle) + self._add_free_block(middle, stop) + return True + else: + return False # too small to record + def _allocate_large_block(self, minsize): # Compute 'size' from 'minsize': it must be rounded up to # 'large_alloc_size'. Additionally, we use the following line @@ -140,6 +156,40 @@ self._allocated = None +class MachineDataBlockWrapper(object): + def __init__(self, asmmemmgr, allblocks): + self.asmmemmgr = asmmemmgr + self.allblocks = allblocks + self.rawstart = 0 + self.rawposition = 0 + self.rawstop = 0 + + def done(self): + if self.rawstart != 0: + if self.asmmemmgr.open_free(self.rawposition, self.rawstop): + self.rawstop = self.rawposition + self.allblocks.append((self.rawstart, self.rawstop)) + self.rawstart = 0 + self.rawposition = 0 + self.rawstop = 0 + + def _allocate_next_block(self, minsize): + self.done() + self.rawstart, self.rawstop = self.asmmemmgr.open_malloc(minsize) + self.rawposition = self.rawstart + + def malloc_aligned(self, size, alignment): + p = self.rawposition + p = (p + alignment - 1) & (-alignment) + if p + size > self.rawstop: + self._allocate_next_block(size + alignment - 1) + p = self.rawposition + p = (p + alignment - 1) & (-alignment) + assert p + size <= self.rawstop + self.rawposition = p + size + return p + + class BlockBuilderMixin(object): _mixin_ = True # A base class to generate assembler. It is equivalent to just a list @@ -156,7 +206,6 @@ SUBBLOCK_PTR.TO.become(SUBBLOCK) gcroot_markers = None - gcroot_markers_total_size = 0 def __init__(self, translated=None): if translated is None: @@ -224,11 +273,8 @@ self.copy_to_raw_memory(rawstart) if self.gcroot_markers is not None: assert gcrootmap is not None - gcrootmap.add_raw_gcroot_markers(asmmemmgr, - allblocks, - self.gcroot_markers, - self.gcroot_markers_total_size, - rawstart) + for pos, mark in self.gcroot_markers: + gcrootmap.put(rawstart + pos, mark) return rawstart def _become_a_plain_block_builder(self): @@ -247,4 +293,3 @@ if self.gcroot_markers is None: self.gcroot_markers = [] self.gcroot_markers.append((self.get_relative_pos(), mark)) - self.gcroot_markers_total_size += len(mark) Modified: pypy/trunk/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/gc.py Wed Dec 1 12:06:57 2010 @@ -222,7 +222,7 @@ LOC_EBP_MINUS = 3 GCMAP_ARRAY = rffi.CArray(lltype.Signed) - CALLSHAPE_ARRAY = rffi.CArray(rffi.UCHAR) + CALLSHAPE_ARRAY_PTR = rffi.CArrayPtr(rffi.UCHAR) def __init__(self): # '_gcmap' is an array of length '_gcmap_maxlength' of addresses. @@ -264,8 +264,7 @@ self._gcmap_sorted = True return sorted - @rgc.no_collect - def _put(self, retaddr, callshapeaddr): + def put(self, retaddr, callshapeaddr): """'retaddr' is the address just after the CALL. 'callshapeaddr' is the address of the raw 'shape' marker. Both addresses are actually integers here.""" @@ -308,37 +307,6 @@ lltype.free(oldgcmap, flavor='raw', track_allocation=False) return j - def add_raw_gcroot_markers(self, asmmemmgr, allblocks, - markers, total_size, rawstart): - """The interface is a bit custom, but this routine writes the - shapes of gcroots (for the GC to use) into raw memory.""" - # xxx so far, we never try to share them. But right now - # the amount of potential sharing would not be too large. - dst = 1 - stop = 0 - for relpos, shape in markers: - # - if dst + len(shape) > stop: - # No more space in the previous raw block, - # allocate a raw block of memory big enough to fit - # as many of the remaining 'shapes' as possible - start, stop = asmmemmgr.malloc(len(shape), total_size) - # add the raw block to 'compiled_loop_token.asmmemmgr_blocks' - allblocks.append((start, stop)) - dst = start - # - # add the entry 'pos_after_call -> dst' to the table - self._put(rawstart + relpos, dst) - # Copy 'shape' into the raw memory, reversing the order - # of the bytes. Similar to compress_callshape() in - # trackgcroot.py. - total_size -= len(shape) - src = len(shape) - 1 - while src >= 0: - rffi.cast(rffi.CCHARP, dst)[0] = shape[src] - dst += 1 - src -= 1 - @rgc.no_collect def freeing_block(self, start, stop): # if [start:stop] is a raw block of assembler, then look up the @@ -409,6 +377,16 @@ assert reg_index > 0 shape.append(chr(self.LOC_REG | (reg_index << 2))) + def compress_callshape(self, shape, datablockwrapper): + # Similar to compress_callshape() in trackgcroot.py. + # Returns an address to raw memory (as an integer). + length = len(shape) + rawaddr = datablockwrapper.malloc_aligned(length, 1) + p = rffi.cast(self.CALLSHAPE_ARRAY_PTR, rawaddr) + for i in range(length): + p[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) + return rawaddr + class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py Wed Dec 1 12:06:57 2010 @@ -1,5 +1,6 @@ import random from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager +from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from pypy.rpython.lltypesystem import lltype, rffi @@ -151,14 +152,10 @@ prev_total = new_total def test_insert_gcroot_marker(self): + puts = [] class FakeGcRootMap: - def add_raw_gcroot_markers(self, asmmemmgr, allblocks, markers, - total_size, rawstart): - self.asmmemmgr = asmmemmgr - self.allblocks = allblocks - self.markers = markers - self.total_size = total_size - self.rawstart = rawstart + def put(self, retaddr, mark): + puts.append((retaddr, mark)) # mc = BlockBuilderMixin() mc.writechar('X') @@ -181,10 +178,8 @@ assert p[4] == 'Z' assert p[5] == 'z' assert allblocks == [(rawstart, rawstart + 6)] - assert gcrootmap.markers == [(2, ['a', 'b', 'c', 'd']), - (4, ['e', 'f', 'g'])] - assert gcrootmap.total_size == 4 + 3 - assert gcrootmap.rawstart == rawstart + assert puts == [(rawstart + 2, ['a', 'b', 'c', 'd']), + (rawstart + 4, ['e', 'f', 'g'])] def test_blockbuildermixin(translated=True): @@ -215,3 +210,41 @@ def test_blockbuildermixin2(): test_blockbuildermixin(translated=False) + +def test_machinedatablock(): + ops = [] + class FakeMemMgr: + _addr = 1597 + def open_malloc(self, minsize): + result = (self._addr, self._addr + 100) + ops.append(('malloc', minsize) + result) + self._addr += 200 + return result + def open_free(self, frm, to): + ops.append(('free', frm, to)) + return to - frm >= 8 + # + allblocks = [] + md = MachineDataBlockWrapper(FakeMemMgr(), allblocks) + p = md.malloc_aligned(26, 16) + assert p == 1600 + assert ops == [('malloc', 26 + 15, 1597, 1697)] + del ops[:] + # + p = md.malloc_aligned(26, 16) + assert p == 1632 + p = md.malloc_aligned(26, 16) + assert p == 1664 + assert allblocks == [] + assert ops == [] + # + p = md.malloc_aligned(27, 16) + assert p == 1808 + assert allblocks == [(1597, 1697)] + assert ops == [('free', 1690, 1697), + ('malloc', 27 + 15, 1797, 1897)] + del ops[:] + # + md.done() + assert allblocks == [(1597, 1697), (1797, 1835)] + assert ops == [('free', 1835, 1897)] Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_gc.py Wed Dec 1 12:06:57 2010 @@ -91,11 +91,27 @@ assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, 4, 8, 12, 16]) + def test_compress_callshape(self): + class FakeDataBlockWrapper: + def malloc_aligned(self, size, alignment): + assert alignment == 1 # here + assert size == 4 + return rffi.cast(lltype.Signed, p) + datablockwrapper = FakeDataBlockWrapper() + p = lltype.malloc(rffi.CArray(lltype.Char), 4, immortal=True) + gcrootmap = GcRootMap_asmgcc() + shape = ['a', 'b', 'c', 'd'] + gcrootmap.compress_callshape(shape, datablockwrapper) + assert p[0] == 'd' + assert p[1] == 'c' + assert p[2] == 'b' + assert p[3] == 'a' + def test_put_basic(self): gcrootmap = GcRootMap_asmgcc() retaddr = 1234567890 shapeaddr = 51627384 - gcrootmap._put(retaddr, shapeaddr) + gcrootmap.put(retaddr, shapeaddr) assert gcrootmap._gcmap[0] == retaddr assert gcrootmap._gcmap[1] == shapeaddr p = rffi.cast(rffi.LONGP, gcrootmap.gcmapstart()) @@ -109,7 +125,7 @@ for i in range(700): shapeaddr = i * 100 + 1 retaddr = 123456789 + i - gcrootmap._put(retaddr, shapeaddr) + gcrootmap.put(retaddr, shapeaddr) for i in range(700): assert gcrootmap._gcmap[i*2+0] == 123456789 + i assert gcrootmap._gcmap[i*2+1] == i * 100 + 1 @@ -126,7 +142,7 @@ for i in range(700): shapeaddr = i * 100 # 0 if i == 0 retaddr = 123456789 + i - gcrootmap._put(retaddr, shapeaddr) + gcrootmap.put(retaddr, shapeaddr) if shapeaddr != 0: expected.append((retaddr, shapeaddr)) # at the first resize, the 0 should be removed @@ -142,72 +158,11 @@ # check that we can again insert 350 entries without a resize oldgcmap = gcrootmap._gcmap for i in range(0, 699, 2): - gcrootmap._put(515151 + i + repeat, 626262 + i) + gcrootmap.put(515151 + i + repeat, 626262 + i) expected.append((515151 + i + repeat, 626262 + i)) assert gcrootmap._gcmap == oldgcmap check() - def test_add_raw_gcroot_markers_maxalloc(self): - class FakeAsmMemMgr: - def malloc(self, minsize, maxsize): - assert minsize == 4 - assert maxsize == 7 - return (prawstart, prawstart + 8) - put = [] - def fakeput(a, b): - put.append((a, b)) - gcrootmap = GcRootMap_asmgcc() - gcrootmap._put = fakeput - memmgr = FakeAsmMemMgr() - allblocks = [] - p = lltype.malloc(rffi.CArray(lltype.Char), 7, immortal=True) - prawstart = rffi.cast(lltype.Signed, p) - gcrootmap.add_raw_gcroot_markers(memmgr, allblocks, - [(2, ['a', 'b', 'c', 'd']), - (4, ['e', 'f', 'g'])], - 4 + 3, 1200000) - assert allblocks == [(prawstart, prawstart + 8)] - assert ''.join([p[i] for i in range(7)]) == 'dcbagfe' - assert put == [(1200002, prawstart), - (1200004, prawstart + 4)] - - def test_add_raw_gcroot_markers_minalloc(self): - class FakeAsmMemMgr: - callnum = 0 - def malloc(self, minsize, maxsize): - self.callnum += 1 - if self.callnum == 1: - assert minsize == 4 - assert maxsize == 7 - return (prawstart, prawstart + 6) - elif self.callnum == 2: - assert minsize == 3 - assert maxsize == 3 - return (qrawstart, qrawstart + 5) - else: - raise AssertionError - put = [] - def fakeput(a, b): - put.append((a, b)) - gcrootmap = GcRootMap_asmgcc() - gcrootmap._put = fakeput - memmgr = FakeAsmMemMgr() - allblocks = [] - p = lltype.malloc(rffi.CArray(lltype.Char), 6, immortal=True) - prawstart = rffi.cast(lltype.Signed, p) - q = lltype.malloc(rffi.CArray(lltype.Char), 5, immortal=True) - qrawstart = rffi.cast(lltype.Signed, q) - gcrootmap.add_raw_gcroot_markers(memmgr, allblocks, - [(2, ['a', 'b', 'c', 'd']), - (4, ['e', 'f', 'g'])], - 4 + 3, 1200000) - assert allblocks == [(prawstart, prawstart + 6), - (qrawstart, qrawstart + 5)] - assert ''.join([p[i] for i in range(4)]) == 'dcba' - assert ''.join([q[i] for i in range(3)]) == 'gfe' - assert put == [(1200002, prawstart), - (1200004, qrawstart)] - def test_freeing_block(self): from pypy.jit.backend.llsupport import gc class Asmgcroot: Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Wed Dec 1 12:06:57 2010 @@ -1,5 +1,6 @@ import sys, os from pypy.jit.backend.llsupport import symbolic +from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT, LoopToken) @@ -55,7 +56,6 @@ DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) class Assembler386(object): - _float_constants = None _regalloc = None _output_loop_log = None @@ -83,6 +83,7 @@ self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') self.fail_boxes_count = 0 self._current_depths_cache = (0, 0) + self.datablockwrapper = None self.teardown() def leave_jitted_hook(self): @@ -125,10 +126,14 @@ self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') - def setup(self): + def setup(self, looptoken): assert self.memcpy_addr != 0, "setup_once() not called?" self.pending_guard_tokens = [] self.mc = codebuf.MachineCodeBlockWrapper() + if self.datablockwrapper is None: + allblocks = self.get_asmmemmgr_blocks(looptoken) + self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, + allblocks) def teardown(self): self.pending_guard_tokens = None @@ -145,13 +150,9 @@ debug_stop('jit-backend-counts') def _build_float_constants(self): - # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment - addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw', - track_allocation=False) - if not we_are_translated(): - self._keepalive_malloced_float_consts = addr - float_constants = rffi.cast(lltype.Signed, addr) - float_constants = (float_constants + 15) & ~15 # align to 16 bytes + datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, []) + float_constants = datablockwrapper.malloc_aligned(32, alignment=16) + datablockwrapper.done() addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' # 0x8000000000000000 @@ -203,13 +204,18 @@ _x86_arglocs _x86_debug_checksum ''' + # XXX this function is too longish and contains some code + # duplication with assemble_bridge(). Also, we should think + # about not storing on 'self' attributes that will live only + # for the duration of compiling one loop or a one bridge. + clt = CompiledLoopToken(self.cpu, looptoken.number) looptoken.compiled_loop_token = clt if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.setup() + self.setup(looptoken) self.currently_compiling_loop = looptoken funcname = self._find_debug_merge_point(operations) if log: @@ -235,7 +241,7 @@ self.write_pending_failure_recoveries() fullsize = self.mc.get_relative_pos() # - rawstart = self.materialize(looptoken) + rawstart = self.materialize_loop(looptoken) debug_print("Loop #%d (%s) has address %x to %x" % ( looptoken.number, funcname, rawstart + self.looppos, @@ -268,7 +274,7 @@ "was already compiled!") return - self.setup() + self.setup(original_loop_token) funcname = self._find_debug_merge_point(operations) if log: self._register_counter() @@ -289,7 +295,7 @@ self.write_pending_failure_recoveries() fullsize = self.mc.get_relative_pos() # - rawstart = self.materialize(original_loop_token) + rawstart = self.materialize_loop(original_loop_token) debug_print("Bridge out of guard %d (%s) has address %x to %x" % (descr_number, funcname, rawstart, rawstart + codeendpos)) @@ -328,12 +334,17 @@ p = rffi.cast(rffi.INTP, addr) p[0] = rffi.cast(rffi.INT, relative_target) - def materialize(self, looptoken): + def get_asmmemmgr_blocks(self, looptoken): clt = looptoken.compiled_loop_token if clt.asmmemmgr_blocks is None: clt.asmmemmgr_blocks = [] - return self.mc.materialize(self.cpu.asmmemmgr, - clt.asmmemmgr_blocks, + return clt.asmmemmgr_blocks + + def materialize_loop(self, looptoken): + self.datablockwrapper.done() # finish using cpu.asmmemmgr + self.datablockwrapper = None + allblocks = self.get_asmmemmgr_blocks(looptoken) + return self.mc.materialize(self.cpu.asmmemmgr, allblocks, self.cpu.gc_ll_descr.gcrootmap) def _find_debug_merge_point(self, operations): Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Wed Dec 1 12:06:57 2010 @@ -60,32 +60,6 @@ r15: 5, } -class FloatConstants(object): - BASE_CONSTANT_SIZE = 1000 - - def __init__(self): - self.cur_array_free = 0 - self.const_id = 0 - - def _get_new_array(self): - n = self.BASE_CONSTANT_SIZE - # known to leak - self.cur_array = lltype.malloc(rffi.CArray(lltype.Float), n, # YYY leak - flavor='raw', track_allocation=False) - self.cur_array_free = n - _get_new_array._dont_inline_ = True - - def record_float(self, floatval): - if self.cur_array_free == 0: - self._get_new_array() - arr = self.cur_array - n = self.cur_array_free - 1 - arr[n] = floatval - self.cur_array_free = n - self.const_id += 1 - return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) - - class X86XMMRegisterManager(RegisterManager): box_types = [FLOAT] @@ -93,20 +67,11 @@ # we never need lower byte I hope save_around_call_regs = all_regs - def __init__(self, longevity, frame_manager=None, assembler=None): - RegisterManager.__init__(self, longevity, frame_manager=frame_manager, - assembler=assembler) - if assembler is None: - self.float_constants = FloatConstants() - else: - if assembler._float_constants is None: - assembler._float_constants = FloatConstants() - self.float_constants = assembler._float_constants - def convert_to_imm(self, c): - const_id, adr = self.float_constants.record_float(c.getfloat()) - return ConstFloatLoc(adr, const_id) - + adr = self.assembler.datablockwrapper.malloc_aligned(8, 8) + rffi.cast(rffi.CArrayPtr(rffi.DOUBLE), adr)[0] = c.getfloat() + return ConstFloatLoc(adr) + def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately @@ -1108,7 +1073,8 @@ if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert reg in self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX gcrootmap.add_callee_save_reg(shape, self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX[reg]) - return shape + return gcrootmap.compress_callshape(shape, + self.assembler.datablockwrapper) def consider_force_token(self, op): loc = self.rm.force_allocate_reg(op.result) Modified: pypy/trunk/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regloc.py Wed Dec 1 12:06:57 2010 @@ -177,24 +177,15 @@ class ConstFloatLoc(AssemblerLocation): # XXX: We have to use this class instead of just AddressLoc because - # AddressLoc is "untyped" and also we to have need some sort of unique - # identifier that we can use in _getregkey (for jump.py) - + # we want a width of 8 (... I think. Check this!) _immutable_ = True - width = 8 - def __init__(self, address, const_id): + def __init__(self, address): self.value = address - self.const_id = const_id def __repr__(self): - return '' % (self.value, self.const_id) - - def _getregkey(self): - # XXX: 1000 is kind of magic: We just don't want to be confused - # with any registers - return 1000 + self.const_id + return '' % (self.value,) def location_code(self): return 'j' Modified: pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_gc_integration.py Wed Dec 1 12:06:57 2010 @@ -33,6 +33,10 @@ def add_callee_save_reg(self, shape, reg_index): index_to_name = { 1: 'ebx', 2: 'esi', 3: 'edi' } shape.append(index_to_name[reg_index]) + def compress_callshape(self, shape, datablockwrapper): + assert datablockwrapper == 'fakedatablockwrapper' + assert shape[0] == 'shape' + return ['compressed'] + shape[1:] class MockGcDescr(GcCache): def get_funcptr_for_new(self): @@ -57,6 +61,7 @@ cpu = CPU(None, None) cpu.setup_once() regalloc = RegAlloc(MockAssembler(cpu, MockGcDescr(False))) + regalloc.assembler.datablockwrapper = 'fakedatablockwrapper' boxes = [BoxPtr() for i in range(len(X86RegisterManager.all_regs))] longevity = {} for box in boxes: @@ -79,7 +84,7 @@ assert len(regalloc.assembler.movs) == 3 # mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) - assert mark[0] == 'shape' + assert mark[0] == 'compressed' base = -WORD * FRAME_FIXED_SIZE expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) From arigo at codespeak.net Wed Dec 1 12:07:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 1 Dec 2010 12:07:10 +0100 (CET) Subject: [pypy-svn] r79706 - pypy/branch/jit-free-asm2 Message-ID: <20101201110710.93C6E282BE0@codespeak.net> Author: arigo Date: Wed Dec 1 12:07:09 2010 New Revision: 79706 Removed: pypy/branch/jit-free-asm2/ Log: Remove merged branch. From antocuni at codespeak.net Wed Dec 1 14:25:58 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 1 Dec 2010 14:25:58 +0100 (CET) Subject: [pypy-svn] r79707 - in pypy/branch/jitypes2/pypy: jit/metainterp/test rlib rlib/test Message-ID: <20101201132558.79444282BDD@codespeak.net> Author: antocuni Date: Wed Dec 1 14:25:51 2010 New Revision: 79707 Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_fficall.py pypy/branch/jitypes2/pypy/rlib/libffi.py pypy/branch/jitypes2/pypy/rlib/test/test_libffi.py Log: add support for single float parameters and result to rlib.libffi. A bit hackish because we have to work around the limitation of the JIT which does not support single floats Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_fficall.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_fficall.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_fficall.py Wed Dec 1 14:25:51 2010 @@ -1,5 +1,6 @@ import py +from pypy.rlib.rarithmetic import r_singlefloat from pypy.rlib.jit import JitDriver, hint from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.libffi import ArgChain @@ -35,7 +36,10 @@ func = hint(func, promote=True) argchain = ArgChain() for argval in args: # this loop is unrolled - argchain.arg(argval) + if type(argval) is r_singlefloat: + argchain.arg_singlefloat(float(argval)) + else: + argchain.arg(argval) res = func.call(argchain, RESULT) n += 1 return res Modified: pypy/branch/jitypes2/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/libffi.py (original) +++ pypy/branch/jitypes2/pypy/rlib/libffi.py Wed Dec 1 14:25:51 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib.objectmodel import specialize, enforceargs, we_are_translated -from pypy.rlib.rarithmetic import intmask, r_uint +from pypy.rlib.rarithmetic import intmask, r_uint, r_singlefloat from pypy.rlib import jit from pypy.rlib import clibffi from pypy.rlib.clibffi import get_libc_name, FUNCFLAG_CDECL, AbstractFuncPtr, \ @@ -40,6 +40,7 @@ """ if ffi_type is types.void: return 'v' elif ffi_type is types.double: return 'f' + elif ffi_type is types.float: return 'f' elif ffi_type is types.pointer: return 'i' # elif ffi_type is types.schar: return 'i' @@ -104,11 +105,21 @@ val = rffi.cast(rffi.LONG, val) elif TYPE is rffi.DOUBLE: cls = FloatArg + elif TYPE is rffi.FLOAT: + raise TypeError, 'r_singlefloat not supported by arg(), use arg_singlefloat()' else: raise TypeError, 'Unsupported argument type: %s' % TYPE self._append(cls(val)) return self + def arg_singlefloat(self, val): + """ + Note: you must pass a python Float (rffi.DOUBLE), not a r_singlefloat + (else the jit complains). Note that if you use single floats, the + call won't be jitted at all. + """ + self._append(SingleFloatArg(val)) + def _append(self, arg): if self.first is None: self.first = self.last = arg @@ -132,7 +143,7 @@ func._push_int(self.intval, ll_args, i) class FloatArg(AbstractArg): - """ An argument holding a float + """ An argument holding a python float (i.e. a C double) """ def __init__(self, floatval): @@ -142,6 +153,17 @@ func._push_float(self.floatval, ll_args, i) +class SingleFloatArg(AbstractArg): + """ An argument holding a C float + """ + + def __init__(self, floatval): + self.floatval = floatval + + def push(self, func, ll_args, i): + func._push_single_float(self.floatval, ll_args, i) + + # ====================================================================== @@ -190,6 +212,10 @@ res = self._do_call_int(self.funcsym, ll_args) elif RESULT is rffi.DOUBLE: return self._do_call_float(self.funcsym, ll_args) + elif RESULT is rffi.FLOAT: + # XXX: even if RESULT is FLOAT, we still return a DOUBLE, else the + # jit complains. Note that the jit is disabled in this case + return self._do_call_single_float(self.funcsym, ll_args) elif RESULT is lltype.Void: return self._do_call_void(self.funcsym, ll_args) else: @@ -223,6 +249,10 @@ def _push_float(self, value, ll_args, i): self._push_arg(value, ll_args, i) + @jit.dont_look_inside + def _push_single_float(self, value, ll_args, i): + self._push_arg(r_singlefloat(value), ll_args, i) + @jit.oopspec('libffi_call_int(self, funcsym, ll_args)') def _do_call_int(self, funcsym, ll_args): return self._do_call(funcsym, ll_args, rffi.LONG) @@ -231,6 +261,11 @@ def _do_call_float(self, funcsym, ll_args): return self._do_call(funcsym, ll_args, rffi.DOUBLE) + @jit.dont_look_inside + def _do_call_single_float(self, funcsym, ll_args): + single_res = self._do_call(funcsym, ll_args, rffi.FLOAT) + return float(single_res) + @jit.oopspec('libffi_call_void(self, funcsym, ll_args)') def _do_call_void(self, funcsym, ll_args): return self._do_call(funcsym, ll_args, lltype.Void) Modified: pypy/branch/jitypes2/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/test/test_libffi.py (original) +++ pypy/branch/jitypes2/pypy/rlib/test/test_libffi.py Wed Dec 1 14:25:51 2010 @@ -2,6 +2,7 @@ import sys from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED +from pypy.rlib.rarithmetic import r_singlefloat from pypy.rlib.test.test_clibffi import BaseFfiTest, get_libm_name from pypy.rlib.libffi import CDLL, Func, get_libc_name, ArgChain, types @@ -110,7 +111,10 @@ func = lib.getpointer(name, argtypes, restype) chain = ArgChain() for arg in args: - chain.arg(arg) + if isinstance(arg, r_singlefloat): + chain.arg_singlefloat(float(arg)) + else: + chain.arg(arg) return func.call(chain, RESULT) def check_loops(self, *args, **kwds): @@ -262,3 +266,19 @@ # res = self.call(get_dummy, [], rffi.LONG) assert res == initval+1 + + def test_single_float_args(self): + """ + float sum_xy_float(float x, float y) + { + return x+y; + } + """ + from ctypes import c_float # this is used only to compute the expected result + libfoo = self.get_libfoo() + func = (libfoo, 'sum_xy_float', [types.float, types.float], types.float) + x = r_singlefloat(12.34) + y = r_singlefloat(56.78) + res = self.call(func, [x, y], rffi.FLOAT, init_result=0.0) + expected = c_float(c_float(12.34).value + c_float(56.78).value).value + assert res == expected From arigo at codespeak.net Wed Dec 1 14:30:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 1 Dec 2010 14:30:20 +0100 (CET) Subject: [pypy-svn] r79708 - pypy/trunk/lib_pypy Message-ID: <20101201133020.70DA2282BDD@codespeak.net> Author: arigo Date: Wed Dec 1 14:30:19 2010 New Revision: 79708 Modified: pypy/trunk/lib_pypy/_marshal.py Log: Revert changes here. The _marshal module is also used by translation.sandbox. Modified: pypy/trunk/lib_pypy/_marshal.py ============================================================================== --- pypy/trunk/lib_pypy/_marshal.py (original) +++ pypy/trunk/lib_pypy/_marshal.py Wed Dec 1 14:30:19 2010 @@ -3,7 +3,7 @@ This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, transport the file to a Sun, and read it back there). Details of the format may change between Python versions. """ -import types, __pypy__ +import types from _codecs import utf_8_decode, utf_8_encode TYPE_NULL = '0' @@ -645,18 +645,15 @@ version = 1 - at __pypy__.builtinify def dump(x, f, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format m = _Marshaller(f.write) m.dump(x) - at __pypy__.builtinify def load(f): um = _Unmarshaller(f.read) return um.load() - at __pypy__.builtinify def dumps(x, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format buffer = [] @@ -664,7 +661,6 @@ m.dump(x) return ''.join(buffer) - at __pypy__.builtinify def loads(s): um = _FastUnmarshaller(s) return um.load() From commits-noreply at bitbucket.org Wed Dec 1 14:32:42 2010 From: commits-noreply at bitbucket.org (commits-noreply at bitbucket.org) Date: Wed, 1 Dec 2010 07:32:42 -0600 (CST) Subject: [pypy-svn] jitviewer commit 9fc4e942ce34: A fix for loops that have no debug_merge_points Message-ID: <20101201133242.D69F1241426@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project jitviewer # URL http://bitbucket.org/pypy/jitviewer/overview # User Maciej Fijalkowski # Date 1291209974 -7200 # Node ID 9fc4e942ce3449023b45a2b6287c9cd39441aaf9 # Parent 18eb76ea4a2af4336a23a344686186d69b83bb12 A fix for loops that have no debug_merge_points --- a/_jitviewer/loops.py +++ b/_jitviewer/loops.py @@ -314,7 +314,7 @@ def slice_debug_merge_points(operations, if not stack: stack.append([]) else: - if bc.inline_level + 1 != len(stack): + if bc.inline_level is not None and bc.inline_level + 1 != len(stack): if bc.inline_level < len(stack): last = stack.pop() stack[-1].append(Function(last, getpath(stack), storage)) From commits-noreply at bitbucket.org Wed Dec 1 14:32:42 2010 From: commits-noreply at bitbucket.org (commits-noreply at bitbucket.org) Date: Wed, 1 Dec 2010 07:32:42 -0600 (CST) Subject: [pypy-svn] jitviewer commit 058dfc821029: merge Message-ID: <20101201133242.E08D9241433@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project jitviewer # URL http://bitbucket.org/pypy/jitviewer/overview # User Maciej Fijalkowski # Date 1291210354 -7200 # Node ID 058dfc8210299219583c17f52a41da1f0b308bb5 # Parent 9fc4e942ce3449023b45a2b6287c9cd39441aaf9 # Parent 6bb33bc846a5baa0e69f535bdcfc497bf2e81965 merge --- a/_jitviewer/loops.py +++ b/_jitviewer/loops.py @@ -55,6 +55,9 @@ class Op(object): def getarg(self, i): return self._getvar(self.args[i]) + def getargs(self): + return [self._getvar(v) for v in self.args] + def getres(self): return self._getvar(self.res) @@ -116,10 +119,11 @@ class Op(object): return '((%s)%s).%s = %s' % (name, self.getarg(0), field, self.getarg(1)) def generic_repr(self): + arglist = ', '.join(self.getargs()) if self.res is not None: - return '%s = %s(%s)' % (self.getres(), self.name, ', '.join(self.args)) + return '%s = %s(%s)' % (self.getres(), self.name, arglist) else: - return '%s(%s)' % (self.name, ', '.join(self.args)) + return '%s(%s)' % (self.name, arglist) def __repr__(self): return '<%s (%s)>' % (self.name, ', '.join([repr(a) From arigo at codespeak.net Wed Dec 1 14:34:04 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 1 Dec 2010 14:34:04 +0100 (CET) Subject: [pypy-svn] r79709 - pypy/trunk/lib_pypy/pypy_test Message-ID: <20101201133404.A0D1A282BE9@codespeak.net> Author: arigo Date: Wed Dec 1 14:34:03 2010 New Revision: 79709 Modified: pypy/trunk/lib_pypy/pypy_test/test_hashlib.py Log: Fix test. Modified: pypy/trunk/lib_pypy/pypy_test/test_hashlib.py ============================================================================== --- pypy/trunk/lib_pypy/pypy_test/test_hashlib.py (original) +++ pypy/trunk/lib_pypy/pypy_test/test_hashlib.py Wed Dec 1 14:34:03 2010 @@ -8,6 +8,15 @@ def test_unicode(): assert isinstance(hashlib.new('sha256', u'xxx'), _hashlib.hash) +pure_python_version = { + 'md5': 'md5.new', + 'sha1': 'sha.new', + 'sha224': '_sha256.sha224', + 'sha256': '_sha256.sha256', + 'sha384': '_sha512.sha384', + 'sha512': '_sha512.sha512', + } + def test_attributes(): for name, expected_size in {'md5': 16, 'sha1': 20, @@ -31,7 +40,9 @@ assert hexdigest == h.hexdigest() # also test the pure Python implementation - h = hashlib.__get_builtin_constructor(name)('') + modname, constructor = pure_python_version[name].split('.') + builder = getattr(__import__(modname), constructor) + h = builder('') assert h.digest_size == expected_size assert h.digestsize == expected_size # From cfbolz at codespeak.net Wed Dec 1 15:06:38 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Wed, 1 Dec 2010 15:06:38 +0100 (CET) Subject: [pypy-svn] r79713 - pypy/extradoc/planning Message-ID: <20101201140638.68BD5282BE9@codespeak.net> Author: cfbolz Date: Wed Dec 1 15:06:37 2010 New Revision: 79713 Modified: pypy/extradoc/planning/jit.txt Log: those things are all done Modified: pypy/extradoc/planning/jit.txt ============================================================================== --- pypy/extradoc/planning/jit.txt (original) +++ pypy/extradoc/planning/jit.txt Wed Dec 1 15:06:37 2010 @@ -82,19 +82,8 @@ This is treated as a megamorphic call (promotion of w_self in typeobject.py) while in fact it is not. -- pypy/objspace/std/inlinedict: put the class into the structure to get - only one promote when using an instance, instead of two: the promotion - of the '.w__class__' and the promotion of the '.structure' - DONE on the better-map-instances branch - - guard_true(frame.is_being_profiled) all over the place -- "op = getattr(llop, spaceop.opname)" at app-level ends up in a long chain - of calls to 'll_streq__rpy_stringPtr_rpy_stringPtr' - -- foo(*args) generates sub-optimal code even if foo takes a fixed number of - arguments, because we always allocate the Arguments class in that case - - xxx (find more examples :-) BACKEND TASKS @@ -115,8 +104,6 @@ LATER (maybe) TASKS ------------------- -- think about code memory management - - think out looking into functions or not, based on arguments, for example contains__Tuple should be unrolled if tuple is of constant length. HARD, blocked by the fact that we don't know constants soon enough From antocuni at codespeak.net Wed Dec 1 15:37:25 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Wed, 1 Dec 2010 15:37:25 +0100 (CET) Subject: [pypy-svn] r79715 - in pypy/branch/jitypes2/pypy: module/_ffi module/_ffi/test rlib Message-ID: <20101201143725.D0A7E282BE3@codespeak.net> Author: antocuni Date: Wed Dec 1 15:37:23 2010 New Revision: 79715 Modified: pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py pypy/branch/jitypes2/pypy/rlib/libffi.py Log: add support for float arguments/return value also at applevel Modified: pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py Wed Dec 1 15:37:23 2010 @@ -79,6 +79,8 @@ argchain.arg(intmask(space.uint_w(w_arg))) elif kind == 'f': argchain.arg(space.float_w(w_arg)) + elif kind == 's': + argchain.arg_singlefloat(space.float_w(w_arg)) else: assert False, "Argument kind '%s' not supported" % kind return argchain @@ -95,6 +97,10 @@ elif reskind == 'f': floatres = self.func.call(argchain, rffi.DOUBLE) return space.wrap(floatres) + elif reskind == 's': + # the result is a float, but widened to be inside a double + floatres = self.func.call(argchain, rffi.FLOAT) + return space.wrap(floatres) else: voidres = self.func.call(argchain, lltype.Void) assert voidres is None Modified: pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py Wed Dec 1 15:37:23 2010 @@ -45,6 +45,13 @@ pow = libm.getpointer('pow', [], types.void) pow_addr = rffi.cast(rffi.LONG, pow.funcsym) cls.w_pow_addr = space.wrap(pow_addr) + # + # these are needed for test_single_float_args + from ctypes import c_float + f_12_34 = c_float(12.34).value + f_56_78 = c_float(56.78).value + f_result = c_float(f_12_34 + f_56_78).value + cls.w_f_12_34_plus_56_78 = space.wrap(f_result) def test_libload(self): import _ffi @@ -119,7 +126,6 @@ return x+y; } """ - import sys from _ffi import CDLL, types libfoo = CDLL(self.libfoo_name) sum_xy = libfoo.getfunc('sum_xy_us', [types.ushort, types.ushort], @@ -127,6 +133,20 @@ assert sum_xy(32000, 8000) == 40000 assert sum_xy(60000, 30000) == 90000 % 65536 + def test_single_float_args(self): + """ + float sum_xy_float(float x, float y) + { + return x+y; + } + """ + from _ffi import CDLL, types + libfoo = CDLL(self.libfoo_name) + sum_xy = libfoo.getfunc('sum_xy_float', [types.float, types.float], + types.float) + res = sum_xy(12.34, 56.78) + assert res == self.f_12_34_plus_56_78 + def test_TypeError_numargs(self): from _ffi import CDLL, types libfoo = CDLL(self.libfoo_name) Modified: pypy/branch/jitypes2/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/libffi.py (original) +++ pypy/branch/jitypes2/pypy/rlib/libffi.py Wed Dec 1 15:37:23 2010 @@ -40,7 +40,7 @@ """ if ffi_type is types.void: return 'v' elif ffi_type is types.double: return 'f' - elif ffi_type is types.float: return 'f' + elif ffi_type is types.float: return 's' elif ffi_type is types.pointer: return 'i' # elif ffi_type is types.schar: return 'i' From arigo at codespeak.net Wed Dec 1 17:17:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 1 Dec 2010 17:17:43 +0100 (CET) Subject: [pypy-svn] r79716 - pypy/trunk/pypy/rpython Message-ID: <20101201161743.799E2282B9D@codespeak.net> Author: arigo Date: Wed Dec 1 17:17:40 2010 New Revision: 79716 Modified: pypy/trunk/pypy/rpython/extfunc.py Log: Add a dont_look_inside. See comment. Modified: pypy/trunk/pypy/rpython/extfunc.py ============================================================================== --- pypy/trunk/pypy/rpython/extfunc.py (original) +++ pypy/trunk/pypy/rpython/extfunc.py Wed Dec 1 17:17:40 2010 @@ -201,6 +201,11 @@ exec py.code.compile(""" from pypy.rlib.objectmodel import running_on_llinterp from pypy.rlib.debug import llinterpcall + from pypy.rlib.jit import dont_look_inside + # note: we say 'dont_look_inside' mostly because the + # JIT does not support 'running_on_llinterp', but in + # theory it is probably right to stop jitting anyway. + @dont_look_inside def ll_wrapper(%s): if running_on_llinterp: return llinterpcall(s_result, fakeimpl, %s) From arigo at codespeak.net Wed Dec 1 17:32:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 1 Dec 2010 17:32:00 +0100 (CET) Subject: [pypy-svn] r79717 - pypy/pysqlite2 Message-ID: <20101201163200.85999282B9D@codespeak.net> Author: arigo Date: Wed Dec 1 17:31:58 2010 New Revision: 79717 Modified: pypy/pysqlite2/__init__.py Log: issue583 resolved Thanks! Modified: pypy/pysqlite2/__init__.py ============================================================================== --- pypy/pysqlite2/__init__.py (original) +++ pypy/pysqlite2/__init__.py Wed Dec 1 17:31:58 2010 @@ -0,0 +1 @@ +from dbapi2 import * From david at codespeak.net Wed Dec 1 17:35:58 2010 From: david at codespeak.net (david at codespeak.net) Date: Wed, 1 Dec 2010 17:35:58 +0100 (CET) Subject: [pypy-svn] r79718 - pypy/branch/arm-backend/pypy/jit/backend/arm/test Message-ID: <20101201163558.86EFC282BE3@codespeak.net> Author: david Date: Wed Dec 1 17:35:57 2010 New Revision: 79718 Added: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_ztranslate_backend.py Log: Test for the translation of the backend Added: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_ztranslate_backend.py ============================================================================== --- (empty file) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_ztranslate_backend.py Wed Dec 1 17:35:57 2010 @@ -0,0 +1,55 @@ +import py +from pypy.jit.metainterp.history import (AbstractFailDescr, + AbstractDescr, + BasicFailDescr, + BoxInt, Box, BoxPtr, + LoopToken, + ConstInt, ConstPtr, + BoxObj, Const, + ConstObj, BoxFloat, ConstFloat) +from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.rpython.test.test_llinterp import interpret +from pypy.jit.backend.detect_cpu import getcpuclass +from pypy.jit.backend.arm.runner import ArmCPU + +class FakeStats(object): + pass +cpu = getcpuclass()(rtyper=None, stats=FakeStats(), translate_support_code=True) +class TestBackendTranslation(object): + def test_compile_bridge(self): + def loop(): + i0 = BoxInt() + i1 = BoxInt() + i2 = BoxInt() + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + looptoken = LoopToken() + operations = [ + ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), + ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), + ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), + ResOperation(rop.JUMP, [i1], None, descr=looptoken), + ] + inputargs = [i0] + operations[2].setfailargs([i1]) + cpu.compile_loop(inputargs, operations, looptoken) + + i1b = BoxInt() + i3 = BoxInt() + bridge = [ + ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3), + ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2), + ResOperation(rop.JUMP, [i1b], None, descr=looptoken), + ] + bridge[1].setfailargs([i1b]) + + cpu.compile_bridge(faildescr1, [i1b], bridge) + + cpu.set_future_value_int(0, 2) + fail = cpu.execute_token(looptoken) + res = cpu.get_latest_value_int(0) + return fail.identifier * 1000 + res + + res = interpret(loop, [], insist=True) + assert res == 2020 + From fijal at codespeak.net Wed Dec 1 17:39:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Wed, 1 Dec 2010 17:39:33 +0100 (CET) Subject: [pypy-svn] r79719 - in pypy/branch/out-of-line-guards/pypy/rpython: . lltypesystem test Message-ID: <20101201163933.92BD6282BEB@codespeak.net> Author: fijal Date: Wed Dec 1 17:39:31 2010 New Revision: 79719 Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py pypy/branch/out-of-line-guards/pypy/rpython/rclass.py pypy/branch/out-of-line-guards/pypy/rpython/rvirtualizable2.py pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py Log: Write down part about having jit_invariant_setfield inserted before setfield Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py Wed Dec 1 17:39:31 2010 @@ -322,6 +322,15 @@ self.iprebuiltinstances = identity_dict() self.lowleveltype = Ptr(self.object_type) self.gcflavor = gcflavor + if self.classdef is None: + self.jit_invariant_fields = [] + else: + jit_inv = self.classdef.classdesc.classdict.get( + '_jit_invariant_fields_') + if jit_inv is not None: + self.jit_invariant_fields = dict.fromkeys(jit_inv.value) + else: + self.jit_invariant_fields = [] def _setup_repr(self, llfields=None, hints=None, adtmeths=None): # NOTE: don't store mutable objects like the dicts below on 'self' @@ -335,12 +344,9 @@ fields['__class__'] = 'typeptr', get_type_repr(self.rtyper) else: myllfields = [] - invariant_fields = self.classdef.classdesc.classdict.get( - '_jit_invariant_fields_') - if invariant_fields is not None: - for field in invariant_fields.value: - myllfields.append(('asmcodes_' + field, - lltype.Ptr(ASMCODE))) + for field in self.jit_invariant_fields: + myllfields.append(('asmcodes_' + field, + lltype.Ptr(ASMCODE))) # instance attributes attrs = self.classdef.attrs.items() attrs.sort() @@ -483,7 +489,7 @@ cname = inputconst(Void, mangled_name) if force_cast: vinst = llops.genop('cast_pointer', [vinst], resulttype=self) - self.hook_access_field(vinst, cname, llops, flags) + self.hook_access_field('getfield', vinst, cname, llops, flags) return llops.genop('getfield', [vinst, cname], resulttype=r) else: if self.classdef is None: @@ -499,7 +505,7 @@ cname = inputconst(Void, mangled_name) if force_cast: vinst = llops.genop('cast_pointer', [vinst], resulttype=self) - self.hook_access_field(vinst, cname, llops, flags) + self.hook_access_field('setfield', vinst, cname, llops, flags) llops.genop('setfield', [vinst, cname, vvalue]) else: if self.classdef is None: @@ -507,8 +513,11 @@ self.rbase.setfield(vinst, attr, vvalue, llops, force_cast=True, flags=flags) - def hook_access_field(self, vinst, cname, llops, flags): - pass # for virtualizables; see rvirtualizable2.py + def hook_access_field(self, op, vinst, cname, llops, flags): + # for virtualizables; see rvirtualizable2.py + if (op == 'setfield' and cname.value.startswith('inst_') and + cname.value[len('inst_'):] in self.jit_invariant_fields): + llops.genop('jit_invariant_setfield', []) def new_instance(self, llops, classcallhop=None): """Build a new instance, without calling __init__.""" Modified: pypy/branch/out-of-line-guards/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/rclass.py Wed Dec 1 17:39:31 2010 @@ -162,7 +162,6 @@ hints = hints.copy() hints['immutable'] = True self.immutable_fields = [] # unless overwritten below - self.jit_invariant_fields = [] if self.classdef.classdesc.lookup('_immutable_fields_') is not None: hints = hints.copy() immutable_fields = self.classdef.classdesc.classdict.get( @@ -171,11 +170,8 @@ self.immutable_fields = immutable_fields.value accessor = FieldListAccessor() hints['immutable_fields'] = accessor - if self.classdef.classdesc.lookup('_jit_invariant_fields_') is not None: + if self.jit_invariant_fields: hints = hints.copy() - invariant_fields = self.classdef.classdesc.classdict.get( - '_jit_invariant_fields_') - self.jit_invariant_fields = invariant_fields.value accessor = FieldListAccessor() hints['jit_invariant_fields'] = accessor return hints Modified: pypy/branch/out-of-line-guards/pypy/rpython/rvirtualizable2.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/rvirtualizable2.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/rvirtualizable2.py Wed Dec 1 17:39:31 2010 @@ -48,7 +48,7 @@ self.set_vable(llops, vptr) return vptr - def hook_access_field(self, vinst, cname, llops, flags): + def hook_access_field(self, op, vinst, cname, llops, flags): #if not flags.get('access_directly'): if cname.value in self.my_redirected_fields: cflags = inputconst(lltype.Void, flags) Modified: pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py Wed Dec 1 17:39:31 2010 @@ -749,23 +749,6 @@ assert accessor.fields == {"inst_x" : "", "inst_y" : "[*]"} or \ accessor.fields == {"ox" : "", "oy" : "[*]"} # for ootype - def test_jit_invariant_fields(self): - class A(object): - - _jit_invariant_fields_ = ['x'] - - def __init__(self, x): - self.x = x - - def f(): - return A(3) - - t, typer, graph = self.gengraph(f, []) - A_TYPE = deref(graph.getreturnvar().concretetype) - accessor = A_TYPE._hints["jit_invariant_fields"] - assert accessor.fields == {'inst_x': 'asmcodes_x'} - assert 'asmcodes_x' in A_TYPE._names - def test_immutable_fields_subclass_1(self): class A(object): _immutable_fields_ = ["x"] @@ -1059,11 +1042,37 @@ assert sorted([u]) == [6] # 32-bit types assert sorted([i, r, d, l]) == [2, 3, 4, 5] # 64-bit types - -class TestOOtype(BaseTestRclass, OORtypeMixin): def test_jit_invariant_fields(self): - py.test.skip("no ootype support yet") + class A(object): + _jit_invariant_fields_ = ['x'] + + def __init__(self, x): + self.x = x + + def f(): + return A(3) + + t, typer, graph = self.gengraph(f, []) + A_TYPE = deref(graph.getreturnvar().concretetype) + accessor = A_TYPE._hints["jit_invariant_fields"] + assert accessor.fields == {'inst_x': 'asmcodes_x'} + assert 'asmcodes_x' in A_TYPE._names + + def test_jit_invariant_setfield(self): + class A(object): + _jit_invariant_fields_ = ['x'] + + def f(): + a = A() + a.x = 3 + + t, typer, graph = self.gengraph(f, []) + block = graph.iterblocks().next() + assert 'jit_invariant_setfield' in [op.opname for op + in block.operations] + +class TestOOtype(BaseTestRclass, OORtypeMixin): def test__del__(self): class A(object): def __init__(self): From afa at codespeak.net Wed Dec 1 18:08:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 1 Dec 2010 18:08:30 +0100 (CET) Subject: [pypy-svn] r79720 - in pypy/branch/fast-forward: . ctypes_configure lib-python lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/distutils/tests lib_pypy lib_pypy/pypy_test pypy pypy/annotation pypy/annotation/test pypy/config pypy/config/test pypy/doc pypy/doc/config pypy/doc/statistic pypy/interpreter pypy/interpreter/test pypy/jit/backend pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/optimizeopt pypy/jit/metainterp/test pypy/jit/tl pypy/jit/tl/spli pypy/jit/tool pypy/jit/tool/test pypy/module/__pypy__ pypy/module/__pypy__/test pypy/module/array/benchmark pypy/module/array/test pypy/module/binascii pypy/module/binascii/test pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/src pypy/module/cpyext/test pypy/module/fcntl/test pypy/module/imp pypy/module/itertools pypy/module/posix pypy/module/posix/test pypy/module/pyexpat/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/sys pypy/objspace/std pypy/rlib pypy/rlib/rsre pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/memory/gc pypy/rpython/memory/gc/test pypy/rpython/memory/gctransform pypy/rpython/module pypy/rpython/test pypy/rpython/tool pypy/tool pypy/translator/c/gcc pypy/translator/c/src pypy/translator/goal pypy/translator/goal/test2 pypy/translator/platform pypy/translator/tool Message-ID: <20101201170830.747C6282BEB@codespeak.net> Author: afa Date: Wed Dec 1 18:08:21 2010 New Revision: 79720 Added: pypy/branch/fast-forward/lib_pypy/pypy_test/hack___pypy__.py - copied unchanged from r79701, pypy/trunk/lib_pypy/pypy_test/hack___pypy__.py pypy/branch/fast-forward/pypy/doc/config/objspace.usemodules.binascii.txt - copied unchanged from r79701, pypy/trunk/pypy/doc/config/objspace.usemodules.binascii.txt pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py - copied unchanged from r79701, pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py - copied unchanged from r79701, pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py pypy/branch/fast-forward/pypy/jit/backend/x86/codebuf.py - copied unchanged from r79701, pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/branch/fast-forward/pypy/jit/tool/cpython.vmrss - copied unchanged from r79701, pypy/trunk/pypy/jit/tool/cpython.vmrss pypy/branch/fast-forward/pypy/jit/tool/test/test_log2gnumeric.py - copied unchanged from r79706, pypy/trunk/pypy/jit/tool/test/test_log2gnumeric.py pypy/branch/fast-forward/pypy/module/binascii/ (props changed) - copied from r79701, pypy/trunk/pypy/module/binascii/ pypy/branch/fast-forward/pypy/module/binascii/__init__.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/__init__.py pypy/branch/fast-forward/pypy/module/binascii/app_binascii.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/app_binascii.py pypy/branch/fast-forward/pypy/module/binascii/interp_base64.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/interp_base64.py pypy/branch/fast-forward/pypy/module/binascii/interp_binascii.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/interp_binascii.py pypy/branch/fast-forward/pypy/module/binascii/interp_crc32.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/interp_crc32.py pypy/branch/fast-forward/pypy/module/binascii/interp_hexlify.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/interp_hexlify.py pypy/branch/fast-forward/pypy/module/binascii/interp_hqx.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/interp_hqx.py pypy/branch/fast-forward/pypy/module/binascii/interp_qp.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/interp_qp.py pypy/branch/fast-forward/pypy/module/binascii/interp_uu.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/interp_uu.py pypy/branch/fast-forward/pypy/module/binascii/test/ (props changed) - copied from r79701, pypy/trunk/pypy/module/binascii/test/ pypy/branch/fast-forward/pypy/module/binascii/test/test_binascii.py - copied unchanged from r79701, pypy/trunk/pypy/module/binascii/test/test_binascii.py pypy/branch/fast-forward/pypy/module/cpyext/include/structseq.h - copied unchanged from r79701, pypy/trunk/pypy/module/cpyext/include/structseq.h pypy/branch/fast-forward/pypy/module/cpyext/src/structseq.c - copied unchanged from r79701, pypy/trunk/pypy/module/cpyext/src/structseq.c pypy/branch/fast-forward/pypy/module/cpyext/test/test_structseq.py - copied unchanged from r79701, pypy/trunk/pypy/module/cpyext/test/test_structseq.py pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_inspector.py - copied unchanged from r79701, pypy/trunk/pypy/rpython/memory/gc/test/test_inspector.py pypy/branch/fast-forward/pypy/translator/platform/freebsd.py - copied unchanged from r79701, pypy/trunk/pypy/translator/platform/freebsd.py Removed: pypy/branch/fast-forward/pypy/doc/config/translation.jit_debug.txt pypy/branch/fast-forward/pypy/translator/platform/freebsd7.py Modified: pypy/branch/fast-forward/ (props changed) pypy/branch/fast-forward/ctypes_configure/configure.py pypy/branch/fast-forward/lib-python/conftest.py pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/msvccompiler.py pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py pypy/branch/fast-forward/lib_pypy/_locale.py pypy/branch/fast-forward/lib_pypy/_marshal.py pypy/branch/fast-forward/lib_pypy/_minimal_curses.py pypy/branch/fast-forward/lib_pypy/_pypy_interact.py pypy/branch/fast-forward/lib_pypy/binascii.py pypy/branch/fast-forward/lib_pypy/cPickle.py pypy/branch/fast-forward/lib_pypy/cmath.py pypy/branch/fast-forward/lib_pypy/ctypes_support.py pypy/branch/fast-forward/lib_pypy/grp.py pypy/branch/fast-forward/lib_pypy/itertools.py pypy/branch/fast-forward/lib_pypy/msvcrt.py pypy/branch/fast-forward/lib_pypy/pwd.py pypy/branch/fast-forward/lib_pypy/pyexpat.py pypy/branch/fast-forward/lib_pypy/pypy_test/test_structseq.py pypy/branch/fast-forward/lib_pypy/readline.py pypy/branch/fast-forward/lib_pypy/resource.py pypy/branch/fast-forward/lib_pypy/syslog.py pypy/branch/fast-forward/pypy/ (props changed) pypy/branch/fast-forward/pypy/annotation/description.py pypy/branch/fast-forward/pypy/annotation/listdef.py pypy/branch/fast-forward/pypy/annotation/test/test_annrpython.py pypy/branch/fast-forward/pypy/annotation/unaryop.py pypy/branch/fast-forward/pypy/config/pypyoption.py pypy/branch/fast-forward/pypy/config/test/test_support.py pypy/branch/fast-forward/pypy/conftest.py pypy/branch/fast-forward/pypy/doc/config/objspace.usemodules.array.txt (props changed) pypy/branch/fast-forward/pypy/doc/cpython_differences.txt pypy/branch/fast-forward/pypy/doc/faq.txt pypy/branch/fast-forward/pypy/doc/index.txt pypy/branch/fast-forward/pypy/doc/release-1.4.0.txt pypy/branch/fast-forward/pypy/doc/sprint-reports.txt pypy/branch/fast-forward/pypy/doc/statistic/release_dates.dat pypy/branch/fast-forward/pypy/doc/statistic/sprint_dates.dat pypy/branch/fast-forward/pypy/interpreter/function.py pypy/branch/fast-forward/pypy/interpreter/gateway.py pypy/branch/fast-forward/pypy/interpreter/pycode.py pypy/branch/fast-forward/pypy/interpreter/test/test_gateway.py pypy/branch/fast-forward/pypy/jit/backend/conftest.py pypy/branch/fast-forward/pypy/jit/backend/detect_cpu.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/gc.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/fast-forward/pypy/jit/backend/model.py pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-forward/pypy/jit/backend/x86/regloc.py pypy/branch/fast-forward/pypy/jit/backend/x86/runner.py pypy/branch/fast-forward/pypy/jit/backend/x86/rx86.py pypy/branch/fast-forward/pypy/jit/backend/x86/support.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_runner.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_zmath.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/fast-forward/pypy/jit/backend/x86/valgrind.py pypy/branch/fast-forward/pypy/jit/codewriter/call.py pypy/branch/fast-forward/pypy/jit/codewriter/jtransform.py pypy/branch/fast-forward/pypy/jit/codewriter/support.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_list.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_void_list.py pypy/branch/fast-forward/pypy/jit/metainterp/history.py pypy/branch/fast-forward/pypy/jit/metainterp/memmgr.py pypy/branch/fast-forward/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_memmgr.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_recursive.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py pypy/branch/fast-forward/pypy/jit/tl/spli/interpreter.py pypy/branch/fast-forward/pypy/jit/tl/tl.py pypy/branch/fast-forward/pypy/jit/tool/log-template.gnumeric pypy/branch/fast-forward/pypy/jit/tool/log2gnumeric.py pypy/branch/fast-forward/pypy/module/__pypy__/__init__.py pypy/branch/fast-forward/pypy/module/__pypy__/interp_magic.py pypy/branch/fast-forward/pypy/module/__pypy__/test/test_special.py pypy/branch/fast-forward/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/fast-forward/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/fast-forward/pypy/module/cpyext/api.py pypy/branch/fast-forward/pypy/module/cpyext/cdatetime.py pypy/branch/fast-forward/pypy/module/cpyext/include/tupleobject.h pypy/branch/fast-forward/pypy/module/cpyext/sequence.py pypy/branch/fast-forward/pypy/module/cpyext/slotdefs.py pypy/branch/fast-forward/pypy/module/cpyext/state.py pypy/branch/fast-forward/pypy/module/cpyext/stubs.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_arraymodule.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_datetime.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_sequence.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_tupleobject.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_typeobject.py pypy/branch/fast-forward/pypy/module/cpyext/tupleobject.py pypy/branch/fast-forward/pypy/module/cpyext/typeobject.py pypy/branch/fast-forward/pypy/module/fcntl/test/test_fcntl.py pypy/branch/fast-forward/pypy/module/imp/importing.py pypy/branch/fast-forward/pypy/module/itertools/interp_itertools.py pypy/branch/fast-forward/pypy/module/posix/__init__.py pypy/branch/fast-forward/pypy/module/posix/interp_posix.py pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py pypy/branch/fast-forward/pypy/module/pyexpat/test/test_parser.py pypy/branch/fast-forward/pypy/module/pypyjit/policy.py pypy/branch/fast-forward/pypy/module/pypyjit/test/test_policy.py pypy/branch/fast-forward/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/fast-forward/pypy/module/sys/__init__.py pypy/branch/fast-forward/pypy/objspace/std/mapdict.py pypy/branch/fast-forward/pypy/objspace/std/objspace.py pypy/branch/fast-forward/pypy/rlib/debug.py pypy/branch/fast-forward/pypy/rlib/libffi.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rlib/rdynload.py pypy/branch/fast-forward/pypy/rlib/rerased.py (props changed) pypy/branch/fast-forward/pypy/rlib/rsre/rsre_core.py pypy/branch/fast-forward/pypy/rlib/test/test_libffi.py pypy/branch/fast-forward/pypy/rlib/test/test_rerased.py (props changed) pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/llmemory.py pypy/branch/fast-forward/pypy/rpython/memory/gc/inspector.py pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/fast-forward/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/fast-forward/pypy/rpython/module/ll_os.py pypy/branch/fast-forward/pypy/rpython/module/ll_time.py pypy/branch/fast-forward/pypy/rpython/rlist.py pypy/branch/fast-forward/pypy/rpython/test/test_rint.py pypy/branch/fast-forward/pypy/rpython/test/test_rlist.py pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py pypy/branch/fast-forward/pypy/tool/ansi_print.py pypy/branch/fast-forward/pypy/tool/logparser.py pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py pypy/branch/fast-forward/pypy/translator/c/src/mem.h pypy/branch/fast-forward/pypy/translator/goal/app_main.py pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py pypy/branch/fast-forward/pypy/translator/platform/__init__.py pypy/branch/fast-forward/pypy/translator/platform/darwin.py pypy/branch/fast-forward/pypy/translator/tool/cbuild.py Log: Merge from trunk svn merge -r79456:79701 ../trunk Modified: pypy/branch/fast-forward/ctypes_configure/configure.py ============================================================================== --- pypy/branch/fast-forward/ctypes_configure/configure.py (original) +++ pypy/branch/fast-forward/ctypes_configure/configure.py Wed Dec 1 18:08:21 2010 @@ -559,6 +559,7 @@ C_HEADER = """ #include #include /* for offsetof() */ +#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); Modified: pypy/branch/fast-forward/lib-python/conftest.py ============================================================================== --- pypy/branch/fast-forward/lib-python/conftest.py (original) +++ pypy/branch/fast-forward/lib-python/conftest.py Wed Dec 1 18:08:21 2010 @@ -126,7 +126,7 @@ RegrTest('test_augassign.py', core=True), RegrTest('test_base64.py'), RegrTest('test_bastion.py'), - RegrTest('test_binascii.py'), + RegrTest('test_binascii.py', usemodules='binascii'), RegrTest('test_binhex.py'), Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/msvccompiler.py Wed Dec 1 18:08:21 2010 @@ -25,17 +25,20 @@ from distutils import log from distutils.util import get_platform -import _winreg - -RegOpenKeyEx = _winreg.OpenKeyEx -RegEnumKey = _winreg.EnumKey -RegEnumValue = _winreg.EnumValue -RegError = _winreg.error - -HKEYS = (_winreg.HKEY_USERS, - _winreg.HKEY_CURRENT_USER, - _winreg.HKEY_LOCAL_MACHINE, - _winreg.HKEY_CLASSES_ROOT) +try: + import _winreg +except ImportError: + pass +else: + RegOpenKeyEx = _winreg.OpenKeyEx + RegEnumKey = _winreg.EnumKey + RegEnumValue = _winreg.EnumValue + RegError = _winreg.error + + HKEYS = (_winreg.HKEY_USERS, + _winreg.HKEY_CURRENT_USER, + _winreg.HKEY_LOCAL_MACHINE, + _winreg.HKEY_CLASSES_ROOT) VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" VSEXPRESS_BASE = r"Software\Microsoft\VCExpress\%0.1f" Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py Wed Dec 1 18:08:21 2010 @@ -11,11 +11,15 @@ else: self.fail("could not find a suitable manifest") +class MsvcCompilerSimplerTestCase(unittest.TestCase): + def test_import_module(self): + from distutils.msvccompiler import MSVCCompiler + def test_suite(): if sys.platform == 'win32': return unittest.makeSuite(MsvcCompilerTestCase) else: - return unittest.TestSuite([]) + return unittest.makeSuite(MsvcCompilerSimplerTestCase) if __name__ == "__main__": unittest.main(defaultTest="test_suite") Modified: pypy/branch/fast-forward/lib_pypy/_locale.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_locale.py (original) +++ pypy/branch/fast-forward/lib_pypy/_locale.py Wed Dec 1 18:08:21 2010 @@ -11,6 +11,8 @@ # load the platform-specific cache made by running locale.ctc.py from ctypes_config_cache._locale_cache import * +import __pypy__ + # Ubuntu Gusty i386 structure class lconv(Structure): @@ -158,6 +160,7 @@ ul = ''.join(ul) string.letters = ul + at __pypy__.builtinify def setlocale(category, locale=None): "(integer,string=None) -> string. Activates/queries locale processing." if locale: @@ -182,6 +185,7 @@ groups.append(0) return groups + at __pypy__.builtinify def localeconv(): "() -> dict. Returns numeric and monetary locale-specific parameters." @@ -215,6 +219,7 @@ } return result + at __pypy__.builtinify def strcoll(s1, s2): "string,string -> int. Compares two strings according to the locale." @@ -233,6 +238,7 @@ # Collate the strings. return _wcscoll(s1, s2) + at __pypy__.builtinify def strxfrm(s): "string -> string. Returns a string that behaves for cmp locale-aware." @@ -246,6 +252,7 @@ _strxfrm(buf, s, n2) return buf.value + at __pypy__.builtinify def getdefaultlocale(): # TODO: Port code from CPython for Windows and Mac OS raise NotImplementedError() @@ -267,26 +274,31 @@ raise ValueError("unsupported langinfo constant") if HAS_LIBINTL: + @__pypy__.builtinify def gettext(msg): """gettext(msg) -> string Return translation of msg.""" return _gettext(msg) + @__pypy__.builtinify def dgettext(domain, msg): """dgettext(domain, msg) -> string Return translation of msg in domain.""" return _dgettext(domain, msg) + @__pypy__.builtinify def dcgettext(domain, msg, category): """dcgettext(domain, msg, category) -> string Return translation of msg in domain and category.""" return _dcgettext(domain, msg, category) + @__pypy__.builtinify def textdomain(domain): """textdomain(domain) -> string Set the C library's textdomain to domain, returning the new domain.""" return _textdomain(domain) + @__pypy__.builtinify def bindtextdomain(domain, dir): """bindtextdomain(domain, dir) -> string Bind the C library's domain to dir.""" @@ -297,6 +309,7 @@ return dirname if HAS_BIND_TEXTDOMAIN_CODESET: + @__pypy__.builtinify def bind_textdomain_codeset(domain, codeset): """bind_textdomain_codeset(domain, codeset) -> string Bind the C library's domain to codeset.""" Modified: pypy/branch/fast-forward/lib_pypy/_marshal.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_marshal.py (original) +++ pypy/branch/fast-forward/lib_pypy/_marshal.py Wed Dec 1 18:08:21 2010 @@ -3,7 +3,7 @@ This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, transport the file to a Sun, and read it back there). Details of the format may change between Python versions. """ -import types +import types, __pypy__ from _codecs import utf_8_decode, utf_8_encode TYPE_NULL = '0' @@ -645,15 +645,18 @@ version = 1 + at __pypy__.builtinify def dump(x, f, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format m = _Marshaller(f.write) m.dump(x) + at __pypy__.builtinify def load(f): um = _Unmarshaller(f.read) return um.load() + at __pypy__.builtinify def dumps(x, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format buffer = [] @@ -661,6 +664,7 @@ m.dump(x) return ''.join(buffer) + at __pypy__.builtinify def loads(s): um = _FastUnmarshaller(s) return um.load() Modified: pypy/branch/fast-forward/lib_pypy/_minimal_curses.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_minimal_curses.py (original) +++ pypy/branch/fast-forward/lib_pypy/_minimal_curses.py Wed Dec 1 18:08:21 2010 @@ -35,18 +35,23 @@ # ____________________________________________________________ +import __pypy__ + + at __pypy__.builtinify def setupterm(termstr, fd): err = ctypes.c_int(0) result = clib.setupterm(termstr, fd, ctypes.byref(err)) if result == ERR: raise error("setupterm() failed (err=%d)" % err.value) + at __pypy__.builtinify def tigetstr(cap): result = clib.tigetstr(cap) if ctypes.cast(result, ctypes.c_void_p).value == ERR: return None return ctypes.cast(result, ctypes.c_char_p).value + at __pypy__.builtinify def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) if result is None: Modified: pypy/branch/fast-forward/lib_pypy/_pypy_interact.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_pypy_interact.py (original) +++ pypy/branch/fast-forward/lib_pypy/_pypy_interact.py Wed Dec 1 18:08:21 2010 @@ -4,6 +4,13 @@ def interactive_console(mainmodule=None): + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, 'ps1'): + sys.ps1 = '>>>> ' + if not hasattr(sys, 'ps2'): + sys.ps2 = '.... ' + # try: from _pypy_irc_topic import some_topic text = "And now for something completely different: ``%s''" % ( @@ -15,6 +22,7 @@ print text except ImportError: pass + # try: from pyrepl.simple_interact import check if not check(): Modified: pypy/branch/fast-forward/lib_pypy/binascii.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/binascii.py (original) +++ pypy/branch/fast-forward/lib_pypy/binascii.py Wed Dec 1 18:08:21 2010 @@ -1,3 +1,9 @@ +"""A pure Python implementation of binascii. + +Rather slow and buggy in corner cases. +PyPy provides an RPython version too. +""" + class Error(Exception): pass @@ -277,7 +283,7 @@ if (c > '~' or c == '=' or (header and c == '_') or - (c == '.' and linelen == 0 and (inp == len(data) or + (c == '.' and linelen == 0 and (inp+1 == len(data) or data[inp+1] == '\n' or data[inp+1] == '\r')) or (not istext and (c == '\r' or c == '\n')) or Modified: pypy/branch/fast-forward/lib_pypy/cPickle.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/cPickle.py (original) +++ pypy/branch/fast-forward/lib_pypy/cPickle.py Wed Dec 1 18:08:21 2010 @@ -4,6 +4,7 @@ from pickle import * from pickle import __doc__, __version__, format_version, compatible_formats +import __pypy__ BadPickleGet = KeyError UnpickleableError = PicklingError @@ -31,9 +32,11 @@ def getvalue(self): return self.__f and self.__f.getvalue() + at __pypy__.builtinify def dump(obj, file, protocol=None): Pickler(file, protocol).dump(obj) + at __pypy__.builtinify def dumps(obj, protocol=None): file = StringIO() Pickler(file, protocol).dump(obj) Modified: pypy/branch/fast-forward/lib_pypy/cmath.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/cmath.py (original) +++ pypy/branch/fast-forward/lib_pypy/cmath.py Wed Dec 1 18:08:21 2010 @@ -5,9 +5,9 @@ # much code borrowed from mathmodule.c -import math +import math, __pypy__ from math import e, pi - + # constants _one = complex(1., 0.) @@ -33,11 +33,13 @@ + at __pypy__.builtinify def phase(x): x = _to_complex(x) return math.atan2(x.imag, x.real) + at __pypy__.builtinify def polar(x): x = _to_complex(x) phi = math.atan2(x.imag, x.real) @@ -45,10 +47,12 @@ return r, phi + at __pypy__.builtinify def rect(r, phi): return complex(r * math.cos(phi), r * math.sin(phi)) + at __pypy__.builtinify def acos(x): """acos(x) @@ -58,6 +62,7 @@ return -(_prodi(log((x+(_i*sqrt((_one-(x*x)))))))) + at __pypy__.builtinify def acosh(x): """acosh(x) @@ -68,6 +73,7 @@ return z+z + at __pypy__.builtinify def asin(x): """asin(x) @@ -80,6 +86,7 @@ return -(_prodi(log((sqrt_1_minus_x_sq+_prodi(x))))) + at __pypy__.builtinify def asinh(x): """asinh(x) @@ -90,6 +97,7 @@ return z+z + at __pypy__.builtinify def atan(x): """atan(x) @@ -99,6 +107,7 @@ return _halfi*log(((_i+x)/(_i-x))) + at __pypy__.builtinify def atanh(x): """atanh(x) @@ -108,6 +117,7 @@ return _half*log((_one+x)/(_one-x)) + at __pypy__.builtinify def cos(x): """cos(x) @@ -119,6 +129,7 @@ return complex(real, imag) + at __pypy__.builtinify def cosh(x): """cosh(x) @@ -130,6 +141,7 @@ return complex(real, imag) + at __pypy__.builtinify def exp(x): """exp(x) @@ -142,6 +154,7 @@ return complex(real, imag) + at __pypy__.builtinify def log(x, base=None): """log(x) @@ -156,6 +169,7 @@ return complex(real, imag) + at __pypy__.builtinify def log10(x): """log10(x) @@ -168,6 +182,7 @@ return complex(real, imag) + at __pypy__.builtinify def sin(x): """sin(x) @@ -179,6 +194,7 @@ return complex(real, imag) + at __pypy__.builtinify def sinh(x): """sinh(x) @@ -190,6 +206,7 @@ return complex(real, imag) + at __pypy__.builtinify def sqrt(x): """sqrt(x) @@ -215,6 +232,7 @@ _sqrt_half = sqrt(_half) + at __pypy__.builtinify def tan(x): """tan(x) @@ -235,6 +253,7 @@ return complex(real, imag) + at __pypy__.builtinify def tanh(x): """tanh(x) Modified: pypy/branch/fast-forward/lib_pypy/ctypes_support.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/ctypes_support.py (original) +++ pypy/branch/fast-forward/lib_pypy/ctypes_support.py Wed Dec 1 18:08:21 2010 @@ -25,7 +25,7 @@ def _where_is_errno(): return standard_c_lib.__errno_location() -elif sys.platform == 'darwin': +elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() Modified: pypy/branch/fast-forward/lib_pypy/grp.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/grp.py (original) +++ pypy/branch/fast-forward/lib_pypy/grp.py Wed Dec 1 18:08:21 2010 @@ -2,7 +2,7 @@ """ This module provides ctypes version of cpython's grp module """ -import sys +import sys, __pypy__ if sys.platform == 'win32': raise ImportError("No grp module on Windows") @@ -64,6 +64,7 @@ return Group(res.contents.gr_name, res.contents.gr_passwd, res.contents.gr_gid, mem) + at __pypy__.builtinify def getgrgid(gid): res = libc.getgrgid(gid) if not res: @@ -71,6 +72,7 @@ raise KeyError(gid) return _group_from_gstruct(res) + at __pypy__.builtinify def getgrnam(name): if not isinstance(name, str): raise TypeError("expected string") @@ -79,6 +81,7 @@ raise KeyError(name) return _group_from_gstruct(res) + at __pypy__.builtinify def getgrall(): libc.setgrent() lst = [] Modified: pypy/branch/fast-forward/lib_pypy/itertools.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/itertools.py (original) +++ pypy/branch/fast-forward/lib_pypy/itertools.py Wed Dec 1 18:08:21 2010 @@ -27,6 +27,8 @@ 'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap', 'takewhile', 'tee'] +import __pypy__ + class chain(object): """Make an iterator that returns elements from the first iterable @@ -631,7 +633,8 @@ def __iter__(self): return self - + + at __pypy__.builtinify def tee(iterable, n=2): """Return n independent iterators from a single iterable. Note : once tee() has made a split, the original iterable Modified: pypy/branch/fast-forward/lib_pypy/msvcrt.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/msvcrt.py (original) +++ pypy/branch/fast-forward/lib_pypy/msvcrt.py Wed Dec 1 18:08:21 2010 @@ -11,6 +11,7 @@ from ctypes_support import standard_c_lib as _c from ctypes_support import get_errno import errno +import __pypy__ try: open_osfhandle = _c._open_osfhandle @@ -34,6 +35,7 @@ _locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] _locking.restype = ctypes.c_int + at __pypy__.builtinify def locking(fd, mode, nbytes): '''lock or unlock a number of bytes in a file.''' rv = _locking(fd, mode, nbytes) Modified: pypy/branch/fast-forward/lib_pypy/pwd.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/pwd.py (original) +++ pypy/branch/fast-forward/lib_pypy/pwd.py Wed Dec 1 18:08:21 2010 @@ -10,7 +10,7 @@ exception is raised if the entry asked for cannot be found. """ -import sys +import sys, __pypy__ if sys.platform == 'win32': raise ImportError("No pwd module on Windows") @@ -79,10 +79,12 @@ _endpwent.argtypes = None _endpwent.restype = None + at __pypy__.builtinify def mkpwent(pw): pw = pw.contents return struct_passwd(pw) + at __pypy__.builtinify def getpwuid(uid): """ getpwuid(uid) -> (pw_name,pw_passwd,pw_uid, @@ -95,6 +97,7 @@ raise KeyError("getpwuid(): uid not found: %s" % uid) return mkpwent(pw) + at __pypy__.builtinify def getpwnam(name): """ getpwnam(name) -> (pw_name,pw_passwd,pw_uid, @@ -109,9 +112,10 @@ raise KeyError("getpwname(): name not found: %s" % name) return mkpwent(pw) + at __pypy__.builtinify def getpwall(): """ - "getpwall() -> list_of_entries + getpwall() -> list_of_entries Return a list of all available password database entries, in arbitrary order. See pwd.__doc__ for more on password database entries. """ Modified: pypy/branch/fast-forward/lib_pypy/pyexpat.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/pyexpat.py (original) +++ pypy/branch/fast-forward/lib_pypy/pyexpat.py Wed Dec 1 18:08:21 2010 @@ -2,7 +2,7 @@ import ctypes import ctypes.util from ctypes import c_char_p, c_int, c_void_p, POINTER, c_char, c_wchar_p -import sys +import sys, __pypy__ # load the platform-specific cache made by running pyexpat.ctc.py from ctypes_config_cache._pyexpat_cache import * @@ -425,9 +425,11 @@ new_parser._set_unknown_encoding_handler() return new_parser + at __pypy__.builtinify def ErrorString(errno): return XML_ErrorString(errno)[:200] + at __pypy__.builtinify def ParserCreate(encoding=None, namespace_separator=None, intern=None): if (not isinstance(encoding, str) and not encoding is None): Modified: pypy/branch/fast-forward/lib_pypy/pypy_test/test_structseq.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/pypy_test/test_structseq.py (original) +++ pypy/branch/fast-forward/lib_pypy/pypy_test/test_structseq.py Wed Dec 1 18:08:21 2010 @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from .._structseq import * +from .._structseq import structseqfield, structseqtype class mydata: Modified: pypy/branch/fast-forward/lib_pypy/readline.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/readline.py (original) +++ pypy/branch/fast-forward/lib_pypy/readline.py Wed Dec 1 18:08:21 2010 @@ -6,8 +6,4 @@ are only stubs at the moment. """ -# Note that PyPy contains also a built-in module 'readline' which will hide -# this one if compiled in. However the built-in module is incomplete; -# don't use it. - from pyrepl.readline import * Modified: pypy/branch/fast-forward/lib_pypy/resource.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/resource.py (original) +++ pypy/branch/fast-forward/lib_pypy/resource.py Wed Dec 1 18:08:21 2010 @@ -1,4 +1,4 @@ -import sys +import sys, __pypy__ if sys.platform == 'win32': raise ImportError('resource module not available for win32') @@ -77,6 +77,7 @@ ru_nvcsw = _structseq.structseqfield(14) ru_nivcsw = _structseq.structseqfield(15) + at __pypy__.builtinify def rlimit_check_bounds(rlim_cur, rlim_max): if rlim_cur > rlim_t_max: raise ValueError("%d does not fit into rlim_t" % rlim_cur) @@ -89,6 +90,7 @@ ("rlim_max", rlim_t), ) + at __pypy__.builtinify def getrusage(who): ru = _struct_rusage() ret = _getrusage(who, byref(ru)) @@ -116,6 +118,7 @@ ru.ru_nivcsw, )) + at __pypy__.builtinify def getrlimit(resource): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -127,6 +130,7 @@ raise error(errno) return (rlim.rlim_cur, rlim.rlim_max) + at __pypy__.builtinify def setrlimit(resource, rlim): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -143,6 +147,7 @@ else: raise error(errno) + at __pypy__.builtinify def getpagesize(): pagesize = 0 if _getpagesize: Modified: pypy/branch/fast-forward/lib_pypy/syslog.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/syslog.py (original) +++ pypy/branch/fast-forward/lib_pypy/syslog.py Wed Dec 1 18:08:21 2010 @@ -5,7 +5,7 @@ syslog facility. """ -import sys +import sys, __pypy__ if sys.platform == 'win32': raise ImportError("No syslog on Windows") @@ -34,9 +34,11 @@ _setlogmask.argtypes = (c_int,) _setlogmask.restype = c_int + at __pypy__.builtinify def openlog(ident, option, facility): _openlog(ident, option, facility) + at __pypy__.builtinify def syslog(arg1, arg2=None): if arg2 is not None: priority, message = arg1, arg2 @@ -44,15 +46,19 @@ priority, message = LOG_INFO, arg1 _syslog(priority, "%s", message) + at __pypy__.builtinify def closelog(): _closelog() + at __pypy__.builtinify def setlogmask(mask): return _setlogmask(mask) + at __pypy__.builtinify def LOG_MASK(pri): return (1 << pri) + at __pypy__.builtinify def LOG_UPTO(pri): return (1 << (pri + 1)) - 1 Modified: pypy/branch/fast-forward/pypy/annotation/description.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/description.py (original) +++ pypy/branch/fast-forward/pypy/annotation/description.py Wed Dec 1 18:08:21 2010 @@ -636,6 +636,24 @@ return self return None + def maybe_return_immutable_list(self, attr, s_result): + # hack: 'x.lst' where lst is listed in _immutable_fields_ as 'lst[*]' + # should really return an immutable list as a result. Implemented + # by changing the result's annotation (but not, of course, doing an + # actual copy in the rtyper). Tested in pypy.rpython.test.test_rlist, + # test_immutable_list_out_of_instance. + search = '%s[*]' % (attr,) + cdesc = self + while cdesc is not None: + if '_immutable_fields_' in cdesc.classdict: + if search in cdesc.classdict['_immutable_fields_'].value: + s_result.listdef.never_resize() + s_copy = s_result.listdef.offspring() + s_copy.listdef.mark_as_immutable() + return s_copy + cdesc = cdesc.basedesc + return s_result # common case + def consider_call_site(bookkeeper, family, descs, args, s_result): from pypy.annotation.model import SomeInstance, SomePBC, s_None if len(descs) == 1: Modified: pypy/branch/fast-forward/pypy/annotation/listdef.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/listdef.py (original) +++ pypy/branch/fast-forward/pypy/annotation/listdef.py Wed Dec 1 18:08:21 2010 @@ -14,7 +14,7 @@ resized = False # True for lists resized after creation range_step = None # the step -- only for lists only created by a range() dont_change_any_more = False # set to True when too late for changes - must_not_mutate = False # list_not_modified_any_more() + immutable = False # for getattr out of _immutable_fields_ = ['attr[*]'] must_not_resize = False # make_sure_not_resized() # what to do if range_step is different in merge. @@ -38,8 +38,7 @@ if not self.mutated: if self.dont_change_any_more: raise TooLateForChange - if self.must_not_mutate: - raise ListChangeUnallowed("mutating list") + self.immutable = False self.mutated = True def resize(self): @@ -71,10 +70,7 @@ # things more general self, other = other, self - if other.must_not_mutate: - if self.mutated: - raise ListChangeUnallowed("list merge with a mutated") - self.must_not_mutate = True + self.immutable &= other.immutable if other.must_not_resize: if self.resized: raise ListChangeUnallowed("list merge with a resized") @@ -189,9 +185,11 @@ self.listitem.generalize(s_value) def __repr__(self): - return '<[%r]%s%s>' % (self.listitem.s_value, + return '<[%r]%s%s%s%s>' % (self.listitem.s_value, self.listitem.mutated and 'm' or '', - self.listitem.resized and 'r' or '') + self.listitem.resized and 'r' or '', + self.listitem.immutable and 'I' or '', + self.listitem.must_not_resize and '!R' or '') def mutate(self): self.listitem.mutate() @@ -205,11 +203,17 @@ raise ListChangeUnallowed("list already resized") self.listitem.must_not_resize = True - def never_mutate(self): + def mark_as_immutable(self): + # Sets the 'immutable' flag. Note that unlike "never resized", + # the immutable flag is only a hint. It is cleared again e.g. + # when we merge with a "normal" list that doesn't have it. It + # is thus expected to live only shortly, mostly for the case + # of writing 'x.list[n]'. self.never_resize() - if self.listitem.mutated: - raise ListChangeUnallowed("list already mutated") - self.listitem.must_not_mutate = True + if not self.listitem.mutated: + self.listitem.immutable = True + #else: it's fine, don't set immutable=True at all (see + # test_can_merge_immutable_list_with_regular_list) MOST_GENERAL_LISTDEF = ListDef(None, SomeObject()) Modified: pypy/branch/fast-forward/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/fast-forward/pypy/annotation/test/test_annrpython.py Wed Dec 1 18:08:21 2010 @@ -3375,23 +3375,69 @@ a.build_types(fn, []) # assert did not raise ListChangeUnallowed - def test_list_not_modified_any_more(self): - from pypy.rlib.debug import list_not_modified_any_more + def test_return_immutable_list(self): + class A: + _immutable_fields_ = 'lst[*]' + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + return a.lst - def pycode(consts): - return list_not_modified_any_more(consts) - def build1(): - return pycode(consts=[1]) - def build2(): - return pycode(consts=[0]) - def fn(): - build1() - build2() + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert s.listdef.listitem.immutable + + def test_immutable_list_is_actually_resized(self): + class A: + _immutable_fields_ = 'lst[*]' + def f(n): + a = A() + l1 = [n] + l1.append(n+1) + a.lst = l1 + return a.lst a = self.RPythonAnnotator() - a.translator.config.translation.list_comprehension_operations = True - a.build_types(fn, []) - # assert did not raise ListChangeUnallowed + py.test.raises(ListChangeUnallowed, a.build_types, f, [int]) + + def test_can_merge_immutable_list_with_regular_list(self): + class A: + _immutable_fields_ = 'lst[*]' + def foo(lst): + pass + + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + if n > 0: + foo(a.lst) + else: + lst = [0] + lst[0] = n + foo(lst) + + a = self.RPythonAnnotator() + a.build_types(f, [int]) + + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + if n > 0: + lst = [0] + lst[0] = n + foo(lst) + else: + foo(a.lst) + + a = self.RPythonAnnotator() + a.build_types(f, [int]) + def g(n): return [0,1,2,n] Modified: pypy/branch/fast-forward/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/unaryop.py (original) +++ pypy/branch/fast-forward/pypy/annotation/unaryop.py Wed Dec 1 18:08:21 2010 @@ -615,6 +615,9 @@ if basedef.classdesc.all_enforced_attrs is not None: if attr in basedef.classdesc.all_enforced_attrs: raise HarmlesslyBlocked("get enforced attr") + elif isinstance(s_result, SomeList): + s_result = ins.classdef.classdesc.maybe_return_immutable_list( + attr, s_result) return s_result return SomeObject() getattr.can_only_throw = [] Modified: pypy/branch/fast-forward/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/fast-forward/pypy/config/pypyoption.py (original) +++ pypy/branch/fast-forward/pypy/config/pypyoption.py Wed Dec 1 18:08:21 2010 @@ -31,7 +31,7 @@ "crypt", "signal", "_rawffi", "termios", "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", - "_bisect", "_multiprocessing", '_warnings'] + "_bisect", "binascii", "_multiprocessing", '_warnings'] )) translation_modules = default_modules.copy() @@ -163,7 +163,6 @@ suggests=[("objspace.allworkingmodules", False)]), BoolOption("geninterp", "specify whether geninterp should be used", - cmdline=None, default=True), BoolOption("logbytecodes", Modified: pypy/branch/fast-forward/pypy/config/test/test_support.py ============================================================================== --- pypy/branch/fast-forward/pypy/config/test/test_support.py (original) +++ pypy/branch/fast-forward/pypy/config/test/test_support.py Wed Dec 1 18:08:21 2010 @@ -4,32 +4,32 @@ import os, sys, py cpuinfo = """ -processor : 0 +processor\t: 0 -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 37 -model name : Intel(R) Core(TM) i7 CPU L 620 @ 2.00GHz -stepping : 2 - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 37 -model name : Intel(R) Core(TM) i7 CPU L 620 @ 2.00GHz -stepping : 2 - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 37 -model name : Intel(R) Core(TM) i7 CPU L 620 @ 2.00GHz -stepping : 2 -cpu MHz : 1199.000 -cache size : 4096 KB -physical id : 0 -siblings : 4 +processor\t: 1 +vendor_id\t: GenuineIntel +cpu family\t: 6 +model\t\t: 37 +model name\t: Intel(R) Core(TM) i7 CPU L 620 @ 2.00GHz +stepping\t: 2 + +processor\t: 2 +vendor_id\t: GenuineIntel +cpu family\t: 6 +model\t\t: 37 +model name\t: Intel(R) Core(TM) i7 CPU L 620 @ 2.00GHz +stepping\t: 2 + +processor\t: 3 +vendor_id\t: GenuineIntel +cpu family\t: 6 +model\t\t: 37 +model name\t: Intel(R) Core(TM) i7 CPU L 620 @ 2.00GHz +stepping\t: 2 +cpu MHz\t\t: 1199.000 +cache size\t: 4096 KB +physical id\t: 0 +siblings\t: 4 """ class FakeEnviron: Modified: pypy/branch/fast-forward/pypy/conftest.py ============================================================================== --- pypy/branch/fast-forward/pypy/conftest.py (original) +++ pypy/branch/fast-forward/pypy/conftest.py Wed Dec 1 18:08:21 2010 @@ -343,13 +343,15 @@ self.runtest_finish() def runtest_open(self): - leakfinder.start_tracking_allocations() + if not getattr(self.obj, 'dont_track_allocations', False): + leakfinder.start_tracking_allocations() def runtest_perform(self): super(PyPyTestFunction, self).runtest() def runtest_close(self): - if leakfinder.TRACK_ALLOCATIONS: + if (not getattr(self.obj, 'dont_track_allocations', False) + and leakfinder.TRACK_ALLOCATIONS): self._pypytest_leaks = leakfinder.stop_tracking_allocations(False) else: # stop_tracking_allocations() already called self._pypytest_leaks = None Modified: pypy/branch/fast-forward/pypy/doc/cpython_differences.txt ============================================================================== --- pypy/branch/fast-forward/pypy/doc/cpython_differences.txt (original) +++ pypy/branch/fast-forward/pypy/doc/cpython_differences.txt Wed Dec 1 18:08:21 2010 @@ -20,16 +20,21 @@ __builtin__ `__pypy__`_ + _ast + _bisect _codecs _lsprof `_minimal_curses`_ _random `_rawffi`_ + _ssl _socket _sre _weakref + array bz2 cStringIO + `cpyext`_ crypt errno exceptions @@ -72,7 +77,7 @@ * Supported by being rewritten in pure Python (possibly using ``ctypes``): see the `lib_pypy/`_ directory. Examples of modules that we - support this way: ``ctypes``, ``array``, ``cPickle``, + support this way: ``ctypes``, ``cPickle``, ``cStringIO``, ``cmath``, ``dbm`` (?), ``datetime``, ``binascii``... Note that some modules are both in there and in the list above; by default, the built-in module is used (but can be disabled @@ -80,11 +85,13 @@ The extension modules (i.e. modules written in C, in the standard CPython) that are neither mentioned above nor in `lib_pypy/`_ are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) .. the nonstandard modules are listed below... .. _`__pypy__`: __pypy__-module.html .. _`_rawffi`: ctypes-implementation.html .. _`_minimal_curses`: config/objspace.usemodules._minimal_curses.html +.. _`cpyext`: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html .. _Stackless: stackless.html @@ -129,12 +136,10 @@ .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html -The built-in function ``id()`` returns numbers that are not addresses -for most of PyPy's garbage collectors. -This is most visible in the default repr: a typical PyPy object can -pretend to be located ``at 0x00000009``. This is just its ``id()``, not -its real address (because an object can move around in some GCs). Calling -``id`` a lot can lead to performance problem. +Using the default GC called ``minimark``, the built-in function ``id()`` +works like it does in CPython. With other GCs it returns numbers that +are not real addresses (because an object can move around several times) +and calling it a lot can lead to performance problem. Note that if you have a long chain of objects, each with a reference to the next one, and each with a __del__, PyPy's GC will perform badly. On Modified: pypy/branch/fast-forward/pypy/doc/faq.txt ============================================================================== --- pypy/branch/fast-forward/pypy/doc/faq.txt (original) +++ pypy/branch/fast-forward/pypy/doc/faq.txt Wed Dec 1 18:08:21 2010 @@ -47,7 +47,7 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has been a part of 1.3 release, but support is still +change) on trunk. It has been a part of 1.4 release, but support is still in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules Modified: pypy/branch/fast-forward/pypy/doc/index.txt ============================================================================== --- pypy/branch/fast-forward/pypy/doc/index.txt (original) +++ pypy/branch/fast-forward/pypy/doc/index.txt Wed Dec 1 18:08:21 2010 @@ -8,7 +8,7 @@ Getting into PyPy ... ============================================= -* `Release 1.3`_: the latest official release +* `Release 1.4`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -56,4 +56,4 @@ .. _`Documentation`: docindex.html .. _`Getting Started`: getting-started.html .. _papers: extradoc.html -.. _`Release 1.3`: http://pypy.org/download.html +.. _`Release 1.4`: http://pypy.org/download.html Modified: pypy/branch/fast-forward/pypy/doc/release-1.4.0.txt ============================================================================== --- pypy/branch/fast-forward/pypy/doc/release-1.4.0.txt (original) +++ pypy/branch/fast-forward/pypy/doc/release-1.4.0.txt Wed Dec 1 18:08:21 2010 @@ -2,27 +2,58 @@ PyPy 1.4: Ouroboros in practice =============================== -Hello. - We're pleased to announce the 1.4 release of PyPy. This is a major breakthrough in our long journey, as PyPy 1.4 is the first PyPy release that can translate -itself faster than CPython. Starting today, we plan to start using PyPy for our -own development. +itself faster than CPython. Starting today, we are using PyPy more for +our every-day development. So may you :) You can download it here: + + http://pypy.org/download.html + +What is PyPy +============ + +PyPy is a very compliant Python interpreter, almost a drop-in replacement +for CPython. It's fast (`pypy 1.4 and cpython 2.6`_ comparison) -Among other features, this release includes numerous performance improvements +Among its new features, this release includes numerous performance improvements (which made fast self-hosting possible), a 64-bit JIT backend, as well as serious stabilization. As of now, we can consider the 32-bit and 64-bit -linux versions of PyPy stable enoughto run in production. +linux versions of PyPy stable enough to run `in production`_. + +Numerous speed achievements are described on `our blog`_. Normalized speed +charts comparing `pypy 1.4 and pypy 1.3`_ as well as `pypy 1.4 and cpython 2.6`_ +are available on benchmark website. For the impatient: yes, we got a lot faster! More highlights =============== -* Virtualenv support: now PyPy is fully compatible with virtualenv_: note that +* PyPy's built-in Just-in-Time compiler is fully transparent and + automatically generated; it now also has very reasonable memory + requirements. The total memory used by a very complex and + long-running process (translating PyPy itself) is within 1.5x to + at most 2x the memory needed by CPython, for a speed-up of 2x. + +* More compact instances. All instances are as compact as if + they had ``__slots__``. This can give programs a big gain in + memory. (In the example of translation above, we already have + carefully placed ``__slots__``, so there is no extra win.) + +* `Virtualenv support`_: now PyPy is fully compatible with virtualenv_: note that to use it, you need a recent version of virtualenv (>= 1.5). * Faster (and JITted) regular expressions - huge boost in speeding up - sre module. + the `re` module. -* Faster (and JITted) calls to functions like map(). +* Other speed improvements, like JITted calls to functions like map(). .. _virtualenv: http://pypi.python.org/pypi/virtualenv +.. _`Virtualenv support`: http://morepypy.blogspot.com/2010/08/using-virtualenv-with-pypy.html +.. _`in production`: http://morepypy.blogspot.com/2010/11/running-large-radio-telescope-software.html +.. _`our blog`: http://morepypy.blogspot.com +.. _`pypy 1.4 and pypy 1.3`: http://speed.pypy.org/comparison/?exe=1%2B41,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=1%2B41&chart=normal+bars +.. _`pypy 1.4 and cpython 2.6`: http://speed.pypy.org/comparison/?exe=2%2B35,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=2%2B35&chart=normal+bars + +Cheers, + +Carl Friedrich Bolz, Antonio Cuni, Maciej Fijalkowski, +Amaury Forgeot d'Arc, Armin Rigo and the PyPy team Modified: pypy/branch/fast-forward/pypy/doc/sprint-reports.txt ============================================================================== --- pypy/branch/fast-forward/pypy/doc/sprint-reports.txt (original) +++ pypy/branch/fast-forward/pypy/doc/sprint-reports.txt Wed Dec 1 18:08:21 2010 @@ -30,6 +30,17 @@ * `D?sseldorf (October 2006)`_ * `Leysin (January 2007)`_ * `Hildesheim (Feb 2007)`_ (also `EU report writing sprint`_) + * `G?teborg (November 2007)`_ + * `Leysin (January 2008)`_ + * `Berlin (May 2008)`_ + * `Vilnius after EuroPython (July 2008)`_ + * `D?sseldorf (August 2008)`_ + * `Wroclaw (February 2009)`_ + * `Leysin (April 2009)`_ + * `G?teborg (August 2009)`_ + * `D?sseldorf (November 2009)`_ + * `CERN (July 2010)`_ + * `D?sseldorf (October 2010)`_ .. _Hildesheim (Feb 2003): http://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html .. _Gothenburg (May 2003): http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt @@ -55,3 +66,15 @@ .. _Hildesheim (Feb 2007): http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/sprint-report.txt .. _`EU report writing sprint`: http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/eu-report-sprint-report.txt .. _`PyCon/Dallas (Feb 2006)`: http://codespeak.net/pypy/extradoc/sprintinfo/pycon06/sprint-report.txt + .. _`G?teborg (November 2007)`: http://morepypy.blogspot.com/2007_11_01_archive.html + .. _`Leysin (January 2008)`: http://morepypy.blogspot.com/2008/01/leysin-winter-sport-sprint-started.html + .. _`Berlin (May 2008)`: http://morepypy.blogspot.com/2008_05_01_archive.html + .. _`Vilnius after EuroPython (July 2008)`: http://morepypy.blogspot.com/2008/07/europython-2008-pypy-talks-and-sprint.html + .. _`D?sseldorf (August 2008)`: http://morepypy.blogspot.com/2008_10_01_archive.html + .. _`Wroclaw (February 2009)`: http://morepypy.blogspot.com/2009/02/wroclaw-2009-sprint-progress-report.html + .. _`Leysin (April 2009)`: http://morepypy.blogspot.com/2009/04/leysin-sprint-report.html + .. _`G?teborg (August 2009)`: http://morepypy.blogspot.com/2009/08/gothenburg-jit-sprint-report.html + .. _`D?sseldorf (November 2009)`: http://morepypy.blogspot.com/2009/11/dusseldorf-sprint-report.html + .. _`CERN (July 2010)`: http://morepypy.blogspot.com/2010/07/cern-sprint-report-wrapping-c-libraries.html + .. _`D?sseldorf (October 2010)`: http://morepypy.blogspot.com/2010/10/dusseldorf-sprint-report-2010.html + Modified: pypy/branch/fast-forward/pypy/doc/statistic/release_dates.dat ============================================================================== --- pypy/branch/fast-forward/pypy/doc/statistic/release_dates.dat (original) +++ pypy/branch/fast-forward/pypy/doc/statistic/release_dates.dat Wed Dec 1 18:08:21 2010 @@ -7,3 +7,7 @@ 2006-06-25,"PyPy 0.9" 2007-02-17,"PyPy 0.99" 2007-03-27,"PyPy 1.0" +2009-04-28,"PyPy 1.1" +2010-03-12,"PyPy 1.2" +2010-06-26,"PyPy 1.3" +2010-11-26,"PyPy 1.4" Modified: pypy/branch/fast-forward/pypy/doc/statistic/sprint_dates.dat ============================================================================== --- pypy/branch/fast-forward/pypy/doc/statistic/sprint_dates.dat (original) +++ pypy/branch/fast-forward/pypy/doc/statistic/sprint_dates.dat Wed Dec 1 18:08:21 2010 @@ -1,26 +1,39 @@ PyPy sprints location, begin, end "Hildesheim",2003-02-17,2003-02-23 -"Gothenburg",2003-05-24,2003-05-31 -"LovainLaNeuve",2003-06-21,2003-06-24 +"G??teborg",2003-05-24,2003-05-31 +"Louvain-la-Neuve",2003-06-21,2003-06-24 "Berlin",2003-09-29,2003-10-04 "Amsterdam",2003-12-14,2003-12-21 -"Europython/Gothenburg",2004-06-01,2004-06-07 +"Europython/G??teborg",2004-06-01,2004-06-07 "Vilnius",2004-11-15,2004-11-23 "Leysin",2005-01-22,2005-01-29 "PyCon/Washington",2005-03-19,2005-03-22 -"Europython/Gothenburg",2005-07-01,2005-07-07 +"Europython/G??teborg",2005-07-01,2005-07-07 "Hildesheim",2005-07-25,2005-07-31 "Heidelberg",2005-08-22,2005-08-29 "Paris",2005-10-10,2005-10-16 -"Gothenburg",2005-12-05,2005-12-11 +"G??teborg",2005-12-05,2005-12-11 "Mallorca",2006-01-23,2006-01-29 "Pycon/Dallas",2006-02-27,2006-03-02 "Louvain-la-Neuve",2006-03-06,2006-03-10 "Japan",2006-04-23,2006-04-29 -"Duesseldorf",2006-06-02,2006-06-09 +"D??sseldorf",2006-06-02,2006-06-09 "Europython/Genf",2006-07-06,2006-07-09 "Limerick",2006-08-21,2006-08-27 -"Duesseldorf",2006-10-30,2006-11-05 +"D??sseldorf",2006-10-30,2006-11-05 "Leysin",2007-01-08,2007-01-14 "Hildesheim",2007-03-01,2007-03-05 +"Hildesheim",2007-03-18,2007-03-23 +"Bern",2007-10-22,2007-10-26 +"G??teborg",2007-11-19,2007-11-25 +"Leysin",2008-01-12,2008-01-19 +"Berlin",2008-05-17,2008-05-22 +"EuroPython/Vilnius",2008-07-10,2008-07-12 +"D??sseldorf",2008-08-05,2008-08-13 +"Wroclaw",2009-02-07,2009-02-14 +"Leysin",2009-04-14,2009-04-21 +"G??teborg",2009-08-18,2009-08-25 +"D??sseldorf",2009-11-06,2009-11-13 +"CERN",2010-07-05,2010-07-09 +"D??sseldorf",2010-10-25,2010-10-31 Modified: pypy/branch/fast-forward/pypy/interpreter/function.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/function.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/function.py Wed Dec 1 18:08:21 2010 @@ -612,11 +612,9 @@ self.w_func_dict = func.w_func_dict self.w_module = func.w_module - def descr_builtinfunction__new__(space, w_subtype, w_func): - func = space.interp_w(Function, w_func) - bltin = space.allocate_instance(BuiltinFunction, w_subtype) - BuiltinFunction.__init__(bltin, func) - return space.wrap(bltin) + def descr_builtinfunction__new__(space, w_subtype): + raise OperationError(space.w_TypeError, + space.wrap("cannot create 'builtin_function' instances")) def descr_function_repr(self): return self.space.wrap('' % (self.name,)) Modified: pypy/branch/fast-forward/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/gateway.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/gateway.py Wed Dec 1 18:08:21 2010 @@ -1083,7 +1083,7 @@ # these decorators are known to return the same function # object, we may ignore them assert '\n' in source - source = source[source.find('\n') + 1:] + source = source[source.find('\n') + 1:].lstrip() assert source.startswith("def "), "can only transform functions" source = source[4:] p = source.find('(') Modified: pypy/branch/fast-forward/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/pycode.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/pycode.py Wed Dec 1 18:08:21 2010 @@ -15,7 +15,7 @@ CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask -from pypy.rlib.debug import make_sure_not_resized, list_not_modified_any_more +from pypy.rlib.debug import make_sure_not_resized from pypy.rlib import jit from pypy.rlib.objectmodel import compute_hash from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT @@ -69,7 +69,7 @@ self.co_stacksize = stacksize self.co_flags = flags self.co_code = code - self.co_consts_w = list_not_modified_any_more(consts) + self.co_consts_w = consts self.co_names_w = [space.new_interned_str(aname) for aname in names] self.co_varnames = varnames self.co_freevars = freevars @@ -269,7 +269,7 @@ dis.dis(co) def fget_co_consts(space, self): - return space.newtuple(self.co_consts_w[:]) + return space.newtuple(self.co_consts_w) def fget_co_names(space, self): return space.newtuple(self.co_names_w) @@ -383,7 +383,7 @@ w(self.co_stacksize), w(self.co_flags), w(self.co_code), - space.newtuple(self.co_consts_w[:]), + space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), Modified: pypy/branch/fast-forward/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/test/test_gateway.py Wed Dec 1 18:08:21 2010 @@ -578,6 +578,11 @@ w_res = space.call_args(w_g, args) assert space.eq_w(w_res, space.wrap((-1, 0))) +class AppTestPyTestMark: + @py.test.mark.unlikely_to_exist + def test_anything(self): + pass + class TestPassThroughArguments: Modified: pypy/branch/fast-forward/pypy/jit/backend/conftest.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/conftest.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/conftest.py Wed Dec 1 18:08:21 2010 @@ -4,6 +4,8 @@ """ import py, random +option = py.test.config.option + def pytest_addoption(parser): group = parser.getgroup('random test options') group.addoption('--random-seed', action="store", type="int", Modified: pypy/branch/fast-forward/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/detect_cpu.py Wed Dec 1 18:08:21 2010 @@ -31,7 +31,8 @@ 'i86pc': 'x86', # Solaris/Intel 'x86': 'x86', # Apple 'Power Macintosh': 'ppc', - 'x86_64': 'x86', + 'x86_64': 'x86', + 'amd64': 'x86' # freebsd }[mach] except KeyError: return mach Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/gc.py Wed Dec 1 18:08:21 2010 @@ -15,6 +15,7 @@ from pypy.jit.backend.llsupport.descr import GcCache, get_field_descr from pypy.jit.backend.llsupport.descr import GcPtrFieldDescr from pypy.jit.backend.llsupport.descr import get_call_descr +from pypy.rpython.memory.gctransform import asmgcroot # ____________________________________________________________ @@ -35,6 +36,8 @@ return False def has_write_barrier_class(self): return None + def freeing_block(self, start, stop): + pass # ____________________________________________________________ @@ -218,50 +221,152 @@ LOC_EBP_PLUS = 2 LOC_EBP_MINUS = 3 - GCMAP_ARRAY = rffi.CArray(llmemory.Address) + GCMAP_ARRAY = rffi.CArray(lltype.Signed) CALLSHAPE_ARRAY = rffi.CArray(rffi.UCHAR) def __init__(self): + # '_gcmap' is an array of length '_gcmap_maxlength' of addresses. + # '_gcmap_curlength' tells how full the array really is. + # The addresses are actually grouped in pairs: + # (addr-after-the-CALL-in-assembler, addr-of-the-call-shape). + # '_gcmap_deadentries' counts pairs marked dead (2nd item is NULL). + # '_gcmap_sorted' is True only if we know the array is sorted. self._gcmap = lltype.nullptr(self.GCMAP_ARRAY) self._gcmap_curlength = 0 self._gcmap_maxlength = 0 + self._gcmap_deadentries = 0 + self._gcmap_sorted = True def initialize(self): # hack hack hack. Remove these lines and see MissingRTypeAttribute # when the rtyper tries to annotate these methods only when GC-ing... self.gcmapstart() self.gcmapend() + self.gcmarksorted() def gcmapstart(self): - return llmemory.cast_ptr_to_adr(self._gcmap) + return rffi.cast(llmemory.Address, self._gcmap) def gcmapend(self): addr = self.gcmapstart() if self._gcmap_curlength: - addr += llmemory.sizeof(llmemory.Address)*self._gcmap_curlength + addr += rffi.sizeof(lltype.Signed) * self._gcmap_curlength + if not we_are_translated() and type(addr) is long: + from pypy.rpython.lltypesystem import ll2ctypes + addr = ll2ctypes._lladdress(addr) # XXX workaround return addr - def put(self, retaddr, callshapeaddr): + def gcmarksorted(self): + # Called by the GC when it is about to sort [gcmapstart():gcmapend()]. + # Returns the previous sortedness flag -- i.e. returns True if it + # is already sorted, False if sorting is needed. + sorted = self._gcmap_sorted + self._gcmap_sorted = True + return sorted + + @rgc.no_collect + def _put(self, retaddr, callshapeaddr): """'retaddr' is the address just after the CALL. - 'callshapeaddr' is the address returned by encode_callshape().""" + 'callshapeaddr' is the address of the raw 'shape' marker. + Both addresses are actually integers here.""" index = self._gcmap_curlength if index + 2 > self._gcmap_maxlength: - self._enlarge_gcmap() + index = self._enlarge_gcmap() self._gcmap[index] = retaddr self._gcmap[index+1] = callshapeaddr self._gcmap_curlength = index + 2 + self._gcmap_sorted = False + @rgc.no_collect def _enlarge_gcmap(self): - newlength = 250 + self._gcmap_maxlength * 2 - newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', - track_allocation=False) # YYY leak oldgcmap = self._gcmap - for i in range(self._gcmap_curlength): - newgcmap[i] = oldgcmap[i] - self._gcmap = newgcmap - self._gcmap_maxlength = newlength - if oldgcmap: - lltype.free(oldgcmap, flavor='raw', track_allocation=False) + if self._gcmap_deadentries * 3 * 2 > self._gcmap_maxlength: + # More than 1/3rd of the entries are dead. Don't actually + # enlarge the gcmap table, but just clean up the dead entries. + newgcmap = oldgcmap + else: + # Normal path: enlarge the array. + newlength = 250 + (self._gcmap_maxlength // 3) * 4 + newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', + track_allocation=False) + self._gcmap_maxlength = newlength + # + j = 0 + i = 0 + end = self._gcmap_curlength + while i < end: + if oldgcmap[i + 1]: + newgcmap[j] = oldgcmap[i] + newgcmap[j + 1] = oldgcmap[i + 1] + j += 2 + i += 2 + self._gcmap_curlength = j + self._gcmap_deadentries = 0 + if oldgcmap != newgcmap: + self._gcmap = newgcmap + if oldgcmap: + lltype.free(oldgcmap, flavor='raw', track_allocation=False) + return j + + def add_raw_gcroot_markers(self, asmmemmgr, allblocks, + markers, total_size, rawstart): + """The interface is a bit custom, but this routine writes the + shapes of gcroots (for the GC to use) into raw memory.""" + # xxx so far, we never try to share them. But right now + # the amount of potential sharing would not be too large. + dst = 1 + stop = 0 + for relpos, shape in markers: + # + if dst + len(shape) > stop: + # No more space in the previous raw block, + # allocate a raw block of memory big enough to fit + # as many of the remaining 'shapes' as possible + start, stop = asmmemmgr.malloc(len(shape), total_size) + # add the raw block to 'compiled_loop_token.asmmemmgr_blocks' + allblocks.append((start, stop)) + dst = start + # + # add the entry 'pos_after_call -> dst' to the table + self._put(rawstart + relpos, dst) + # Copy 'shape' into the raw memory, reversing the order + # of the bytes. Similar to compress_callshape() in + # trackgcroot.py. + total_size -= len(shape) + src = len(shape) - 1 + while src >= 0: + rffi.cast(rffi.CCHARP, dst)[0] = shape[src] + dst += 1 + src -= 1 + + @rgc.no_collect + def freeing_block(self, start, stop): + # if [start:stop] is a raw block of assembler, then look up the + # corresponding gcroot markers, and mark them as freed now in + # self._gcmap by setting the 2nd address of every entry to NULL. + gcmapstart = self.gcmapstart() + gcmapend = self.gcmapend() + if gcmapstart == gcmapend: + return + if not self.gcmarksorted(): + asmgcroot.sort_gcmap(gcmapstart, gcmapend) + # A note about gcmarksorted(): the deletion we do here keeps the + # array sorted. This avoids needing too many sort_gcmap()s. + # Indeed, freeing_block() is typically called many times in a row, + # so it will call sort_gcmap() at most the first time. + startaddr = rffi.cast(llmemory.Address, start) + stopaddr = rffi.cast(llmemory.Address, stop) + item = asmgcroot.binary_search(gcmapstart, gcmapend, startaddr) + # 'item' points to one of the entries. Because the whole array + # is sorted, we know that it points either to the first entry we + # want to kill, or to the previous entry. + if item.address[0] < startaddr: + item += asmgcroot.arrayitemsize # go forward one entry + assert item == gcmapend or item.address[0] >= startaddr + while item != gcmapend and item.address[0] < stopaddr: + item.address[1] = llmemory.NULL + self._gcmap_deadentries += 1 + item += asmgcroot.arrayitemsize def get_basic_shape(self, is_64_bit=False): # XXX: Should this code even really know about stack frame layout of @@ -304,18 +409,6 @@ assert reg_index > 0 shape.append(chr(self.LOC_REG | (reg_index << 2))) - def compress_callshape(self, shape): - # Similar to compress_callshape() in trackgcroot.py. - # XXX so far, we always allocate a new small array (we could regroup - # them inside bigger arrays) and we never try to share them. - length = len(shape) - compressed = lltype.malloc(self.CALLSHAPE_ARRAY, length, - flavor='raw', - track_allocation=False) # YYY leak - for i in range(length): - compressed[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) - return llmemory.cast_ptr_to_adr(compressed) - class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): @@ -379,6 +472,7 @@ 'layoutbuilder': self.layoutbuilder, 'gcmapstart': lambda: gcrootmap.gcmapstart(), 'gcmapend': lambda: gcrootmap.gcmapend(), + 'gcmarksorted': lambda: gcrootmap.gcmarksorted(), } self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc @@ -641,6 +735,9 @@ def has_write_barrier_class(self): return WriteBarrierDescr + def freeing_block(self, start, stop): + self.gcrootmap.freeing_block(start, stop) + # ____________________________________________________________ def get_ll_description(gcdescr, translator=None, rtyper=None): Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py Wed Dec 1 18:08:21 2010 @@ -18,6 +18,7 @@ from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr from pypy.jit.backend.llsupport.ffisupport import get_call_descr_dynamic +from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -52,6 +53,7 @@ else: self._setup_exception_handling_untranslated() self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) + self.asmmemmgr = AsmMemoryManager() self.setup() if translate_support_code: self._setup_on_leave_jitted_translated() @@ -177,6 +179,15 @@ self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) return exc + def free_loop_and_bridges(self, compiled_loop_token): + AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) + blocks = compiled_loop_token.asmmemmgr_blocks + if blocks is not None: + compiled_loop_token.asmmemmgr_blocks = None + for rawstart, rawstop in blocks: + self.gc_ll_descr.freeing_block(rawstart, rawstop) + self.asmmemmgr.free(rawstart, rawstop) + # ------------------- helpers and descriptions -------------------- @staticmethod Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_gc.py Wed Dec 1 18:08:21 2010 @@ -9,6 +9,7 @@ from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.jit.metainterp.test.test_optimizeopt import equaloplists +from pypy.rpython.memory.gctransform import asmgcroot def test_boehm(): gc_ll_descr = GcLLDescr_boehm(None, None, None) @@ -62,58 +63,214 @@ for i in range(len(allocs)): assert addrs[i].address[0] == llmemory.cast_ptr_to_adr(allocs[i]) -def test_GcRootMap_asmgcc(): - def frame_pos(n): - return -4*(4+n) - gcrootmap = GcRootMap_asmgcc() - num1 = frame_pos(-5) - num1a = num1|2 - num2 = frame_pos(55) - num2a = ((-num2|3) >> 7) | 128 - num2b = (-num2|3) & 127 - shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, num1) - gcrootmap.add_ebp_offset(shape, num2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) - gcrootmap.add_callee_save_reg(shape, 1) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4]) - gcrootmap.add_callee_save_reg(shape, 2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8]) - gcrootmap.add_callee_save_reg(shape, 3) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12]) - gcrootmap.add_callee_save_reg(shape, 4) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12, 16]) - # - shapeaddr = gcrootmap.compress_callshape(shape) - PCALLSHAPE = lltype.Ptr(GcRootMap_asmgcc.CALLSHAPE_ARRAY) - p = llmemory.cast_adr_to_ptr(shapeaddr, PCALLSHAPE) - for i, expected in enumerate([16, 12, 8, 4, - num2a, num2b, num1a, 0, 2, 15, 11, 7, 6]): - assert p[i] == expected - # - retaddr = rffi.cast(llmemory.Address, 1234567890) - gcrootmap.put(retaddr, shapeaddr) - assert gcrootmap._gcmap[0] == retaddr - assert gcrootmap._gcmap[1] == shapeaddr - assert gcrootmap.gcmapstart().address[0] == retaddr - # - # the same as before, but enough times to trigger a few resizes - expected_shapeaddr = {} - for i in range(1, 700): +class TestGcRootMapAsmGcc: + + def test_make_shapes(self): + def frame_pos(n): + return -4*(4+n) + gcrootmap = GcRootMap_asmgcc() + num1 = frame_pos(-5) + num1a = num1|2 + num2 = frame_pos(55) + num2a = ((-num2|3) >> 7) | 128 + num2b = (-num2|3) & 127 shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, frame_pos(i)) - shapeaddr = gcrootmap.compress_callshape(shape) - expected_shapeaddr[i] = shapeaddr - retaddr = rffi.cast(llmemory.Address, 123456789 + i) - gcrootmap.put(retaddr, shapeaddr) - for i in range(1, 700): - expected_retaddr = rffi.cast(llmemory.Address, 123456789 + i) - assert gcrootmap._gcmap[i*2+0] == expected_retaddr - assert gcrootmap._gcmap[i*2+1] == expected_shapeaddr[i] + gcrootmap.add_ebp_offset(shape, num1) + gcrootmap.add_ebp_offset(shape, num2) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) + gcrootmap.add_callee_save_reg(shape, 1) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4]) + gcrootmap.add_callee_save_reg(shape, 2) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8]) + gcrootmap.add_callee_save_reg(shape, 3) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8, 12]) + gcrootmap.add_callee_save_reg(shape, 4) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8, 12, 16]) + + def test_put_basic(self): + gcrootmap = GcRootMap_asmgcc() + retaddr = 1234567890 + shapeaddr = 51627384 + gcrootmap._put(retaddr, shapeaddr) + assert gcrootmap._gcmap[0] == retaddr + assert gcrootmap._gcmap[1] == shapeaddr + p = rffi.cast(rffi.LONGP, gcrootmap.gcmapstart()) + assert p[0] == retaddr + assert (gcrootmap.gcmapend() == + gcrootmap.gcmapstart() + rffi.sizeof(lltype.Signed) * 2) + + def test_put_resize(self): + # the same as before, but enough times to trigger a few resizes + gcrootmap = GcRootMap_asmgcc() + for i in range(700): + shapeaddr = i * 100 + 1 + retaddr = 123456789 + i + gcrootmap._put(retaddr, shapeaddr) + for i in range(700): + assert gcrootmap._gcmap[i*2+0] == 123456789 + i + assert gcrootmap._gcmap[i*2+1] == i * 100 + 1 + + def test_remove_nulls(self): + expected = [] + def check(): + assert gcrootmap._gcmap_curlength == len(expected) * 2 + for i, (a, b) in enumerate(expected): + assert gcrootmap._gcmap[i*2] == a + assert gcrootmap._gcmap[i*2+1] == b + # + gcrootmap = GcRootMap_asmgcc() + for i in range(700): + shapeaddr = i * 100 # 0 if i == 0 + retaddr = 123456789 + i + gcrootmap._put(retaddr, shapeaddr) + if shapeaddr != 0: + expected.append((retaddr, shapeaddr)) + # at the first resize, the 0 should be removed + check() + for repeat in range(10): + # now clear up half the entries + assert len(expected) == 699 + for i in range(0, len(expected), 2): + gcrootmap._gcmap[i*2+1] = 0 + gcrootmap._gcmap_deadentries += 1 + expected = expected[1::2] + assert gcrootmap._gcmap_deadentries*6 > gcrootmap._gcmap_maxlength + # check that we can again insert 350 entries without a resize + oldgcmap = gcrootmap._gcmap + for i in range(0, 699, 2): + gcrootmap._put(515151 + i + repeat, 626262 + i) + expected.append((515151 + i + repeat, 626262 + i)) + assert gcrootmap._gcmap == oldgcmap + check() + + def test_add_raw_gcroot_markers_maxalloc(self): + class FakeAsmMemMgr: + def malloc(self, minsize, maxsize): + assert minsize == 4 + assert maxsize == 7 + return (prawstart, prawstart + 8) + put = [] + def fakeput(a, b): + put.append((a, b)) + gcrootmap = GcRootMap_asmgcc() + gcrootmap._put = fakeput + memmgr = FakeAsmMemMgr() + allblocks = [] + p = lltype.malloc(rffi.CArray(lltype.Char), 7, immortal=True) + prawstart = rffi.cast(lltype.Signed, p) + gcrootmap.add_raw_gcroot_markers(memmgr, allblocks, + [(2, ['a', 'b', 'c', 'd']), + (4, ['e', 'f', 'g'])], + 4 + 3, 1200000) + assert allblocks == [(prawstart, prawstart + 8)] + assert ''.join([p[i] for i in range(7)]) == 'dcbagfe' + assert put == [(1200002, prawstart), + (1200004, prawstart + 4)] + + def test_add_raw_gcroot_markers_minalloc(self): + class FakeAsmMemMgr: + callnum = 0 + def malloc(self, minsize, maxsize): + self.callnum += 1 + if self.callnum == 1: + assert minsize == 4 + assert maxsize == 7 + return (prawstart, prawstart + 6) + elif self.callnum == 2: + assert minsize == 3 + assert maxsize == 3 + return (qrawstart, qrawstart + 5) + else: + raise AssertionError + put = [] + def fakeput(a, b): + put.append((a, b)) + gcrootmap = GcRootMap_asmgcc() + gcrootmap._put = fakeput + memmgr = FakeAsmMemMgr() + allblocks = [] + p = lltype.malloc(rffi.CArray(lltype.Char), 6, immortal=True) + prawstart = rffi.cast(lltype.Signed, p) + q = lltype.malloc(rffi.CArray(lltype.Char), 5, immortal=True) + qrawstart = rffi.cast(lltype.Signed, q) + gcrootmap.add_raw_gcroot_markers(memmgr, allblocks, + [(2, ['a', 'b', 'c', 'd']), + (4, ['e', 'f', 'g'])], + 4 + 3, 1200000) + assert allblocks == [(prawstart, prawstart + 6), + (qrawstart, qrawstart + 5)] + assert ''.join([p[i] for i in range(4)]) == 'dcba' + assert ''.join([q[i] for i in range(3)]) == 'gfe' + assert put == [(1200002, prawstart), + (1200004, qrawstart)] + + def test_freeing_block(self): + from pypy.jit.backend.llsupport import gc + class Asmgcroot: + arrayitemsize = 2 * llmemory.sizeof(llmemory.Address) + sort_count = 0 + def sort_gcmap(self, gcmapstart, gcmapend): + self.sort_count += 1 + def binary_search(self, gcmapstart, gcmapend, startaddr): + i = 0 + while (i < gcrootmap._gcmap_curlength//2 and + gcrootmap._gcmap[i*2] < startaddr): + i += 1 + if i > 0: + i -= 1 + assert 0 <= i < gcrootmap._gcmap_curlength//2 + p = rffi.cast(rffi.CArrayPtr(llmemory.Address), gcmapstart) + p = rffi.ptradd(p, 2*i) + return llmemory.cast_ptr_to_adr(p) + saved = gc.asmgcroot + try: + gc.asmgcroot = Asmgcroot() + # + gcrootmap = GcRootMap_asmgcc() + gcrootmap._gcmap = lltype.malloc(gcrootmap.GCMAP_ARRAY, + 1400, flavor='raw', + immortal=True) + for i in range(700): + gcrootmap._gcmap[i*2] = 1200000 + i + gcrootmap._gcmap[i*2+1] = i * 100 + 1 + assert gcrootmap._gcmap_deadentries == 0 + assert gc.asmgcroot.sort_count == 0 + gcrootmap._gcmap_maxlength = 1400 + gcrootmap._gcmap_curlength = 1400 + gcrootmap._gcmap_sorted = False + # + gcrootmap.freeing_block(1200000 - 100, 1200000) + assert gcrootmap._gcmap_deadentries == 0 + assert gc.asmgcroot.sort_count == 1 + # + gcrootmap.freeing_block(1200000 + 100, 1200000 + 200) + assert gcrootmap._gcmap_deadentries == 100 + assert gc.asmgcroot.sort_count == 1 + for i in range(700): + if 100 <= i < 200: + expected = 0 + else: + expected = i * 100 + 1 + assert gcrootmap._gcmap[i*2] == 1200000 + i + assert gcrootmap._gcmap[i*2+1] == expected + # + gcrootmap.freeing_block(1200000 + 650, 1200000 + 750) + assert gcrootmap._gcmap_deadentries == 150 + assert gc.asmgcroot.sort_count == 1 + for i in range(700): + if 100 <= i < 200 or 650 <= i: + expected = 0 + else: + expected = i * 100 + 1 + assert gcrootmap._gcmap[i*2] == 1200000 + i + assert gcrootmap._gcmap[i*2+1] == expected + # + finally: + gc.asmgcroot = saved class FakeLLOp(object): Modified: pypy/branch/fast-forward/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/model.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/model.py Wed Dec 1 18:08:21 2010 @@ -139,10 +139,14 @@ # resume descrs are the largest consumers of memory (about 3x # more than the assembler, in the case of the x86 backend). lst = self.fail_descr_list - for n in compiled_loop_token.faildescr_indices: + # We expect 'compiled_loop_token' to be itself garbage-collected soon, + # but better safe than sorry: be ready to handle several calls to + # free_loop_and_bridges() for the same compiled_loop_token. + faildescr_indices = compiled_loop_token.faildescr_indices + compiled_loop_token.faildescr_indices = [] + for n in faildescr_indices: lst[n] = None - self.fail_descr_free_list.extend(compiled_loop_token.faildescr_indices) - # We expect 'compiled_loop_token' to be itself garbage-collected soon. + self.fail_descr_free_list.extend(faildescr_indices) @staticmethod def sizeof(S): @@ -271,6 +275,9 @@ class CompiledLoopToken(object): + asmmemmgr_blocks = None + asmmemmgr_gcroots = 0 + def __init__(self, cpu, number): cpu.total_compiled_loops += 1 self.cpu = cpu Modified: pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py Wed Dec 1 18:08:21 2010 @@ -211,6 +211,7 @@ assert self.cpu.total_compiled_loops == 1 assert self.cpu.total_compiled_bridges == 1 + return looptoken def test_compile_bridge_with_holes(self): i0 = BoxInt() @@ -2229,6 +2230,20 @@ assert res.value == expected, ( "%r: got %r, expected %r" % (RESTYPE, res.value, expected)) + def test_free_loop_and_bridges(self): + from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("not a subclass of llmodel.AbstractLLCPU") + if hasattr(self.cpu, 'setup_once'): + self.cpu.setup_once() + mem0 = self.cpu.asmmemmgr.total_mallocs + looptoken = self.test_compile_bridge() + mem1 = self.cpu.asmmemmgr.total_mallocs + self.cpu.free_loop_and_bridges(looptoken.compiled_loop_token) + mem2 = self.cpu.asmmemmgr.total_mallocs + assert mem2 < mem1 + assert mem2 == mem0 + class OOtypeBackendTest(BaseBackendTest): Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py Wed Dec 1 18:08:21 2010 @@ -6,7 +6,6 @@ from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper -from pypy.tool.uid import fixid from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, @@ -31,7 +30,9 @@ from pypy.jit.backend.x86 import rx86, regloc, codebuf from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array -from pypy.rlib.debug import debug_print, debug_start, debug_stop +from pypy.jit.backend.x86 import support +from pypy.rlib.debug import (debug_print, debug_start, debug_stop, + have_debug_prints) from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.jit.metainterp.history import ConstInt, BoxInt @@ -43,122 +44,17 @@ def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) -class MachineCodeBlockWrapper(object): - MC_DEFAULT_SIZE = 1024*1024 - - def __init__(self, assembler, bigsize, profile_agent=None): - self.assembler = assembler - self.old_mcs = [] # keepalive - self.bigsize = bigsize - self._mc = self._instantiate_mc() - self.function_name = None - self.profile_agent = profile_agent - self.reset_reserved_bytes() - - def _instantiate_mc(self): # hook for testing - return codebuf.MachineCodeBlock(self.bigsize) - - def ensure_bytes_available(self, num_bytes): - if self.bytes_free() <= (self._reserved_bytes + num_bytes): - self.make_new_mc() - - def reserve_bytes(self, num_bytes): - self.ensure_bytes_available(num_bytes) - self._reserved_bytes += num_bytes - - def reset_reserved_bytes(self): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - self._reserved_bytes = 64 - - def get_relative_pos(self): - return self._mc.get_relative_pos() - - def overwrite(self, pos, listofchars): - return self._mc.overwrite(pos, listofchars) - - def bytes_free(self): - return self._mc._size - self._mc.get_relative_pos() - - def start_function(self, name): - self.function_name = name - self.start_pos = self._mc.get_relative_pos() - - def end_function(self, done=True): - assert self.function_name is not None - size = self._mc.get_relative_pos() - self.start_pos - address = self.tell() - size - if self.profile_agent is not None: - self.profile_agent.native_code_written(self.function_name, - address, size) - if done: - self.function_name = None - - def make_new_mc(self): - new_mc = self._instantiate_mc() - debug_print('[new machine code block at', new_mc.tell(), ']') - - if IS_X86_64: - # The scratch register is sometimes used as a temporary - # register, but the JMP below might clobber it. Rather than risk - # subtle bugs, we preserve the scratch register across the jump. - self._mc.PUSH_r(X86_64_SCRATCH_REG.value) - - self._mc.JMP(imm(new_mc.tell())) - - if IS_X86_64: - # Restore scratch reg - new_mc.POP_r(X86_64_SCRATCH_REG.value) - - if self.function_name is not None: - self.end_function(done=False) - self.start_pos = new_mc.get_relative_pos() - - self.assembler.write_pending_failure_recoveries() - - self._mc.done() - self.old_mcs.append(self._mc) - self._mc = new_mc - make_new_mc._dont_inline_ = True - - def tell(self): - return self._mc.tell() - - def done(self): - self._mc.done() - -def _new_method(name): - def method(self, *args): - if self.bytes_free() < self._reserved_bytes: - self.make_new_mc() - getattr(self._mc, name)(*args) - method.func_name = name - return method - -for _name in rx86.all_instructions + regloc.all_extra_instructions: - setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) - -for name in dir(codebuf.MachineCodeBlock): - if name.upper() == name or name == "writechr": - setattr(MachineCodeBlockWrapper, name, _new_method(name)) class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + def __init__(self, faildescr, failargs, fail_locs, exc): self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs self.exc = exc - self.desc_bytes = desc_bytes - - def recovery_stub_size(self): - # XXX: 32 is pulled out of the air - return 32 + len(self.desc_bytes) DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) class Assembler386(object): - mc = None - mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE _float_constants = None _regalloc = None _output_loop_log = None @@ -177,16 +73,17 @@ self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 self.loop_run_counters = [] - # if we have 10000 loops, we have some other problems I guess self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 - self.pending_guard_tokens = None self.memcpy_addr = 0 self.setup_failure_recovery() self._debug = False self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.fail_boxes_count = 0 + self._current_depths_cache = (0, 0) + self.teardown() def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -196,54 +93,55 @@ def set_debug(self, v): self._debug = v + def setup_once(self): + # the address of the function called by 'new' + gc_ll_descr = self.cpu.gc_ll_descr + gc_ll_descr.initialize() + ll_new = gc_ll_descr.get_funcptr_for_new() + self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) + if gc_ll_descr.get_funcptr_for_newarray is not None: + ll_new_array = gc_ll_descr.get_funcptr_for_newarray() + self.malloc_array_func_addr = rffi.cast(lltype.Signed, + ll_new_array) + if gc_ll_descr.get_funcptr_for_newstr is not None: + ll_new_str = gc_ll_descr.get_funcptr_for_newstr() + self.malloc_str_func_addr = rffi.cast(lltype.Signed, + ll_new_str) + if gc_ll_descr.get_funcptr_for_newunicode is not None: + ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() + self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, + ll_new_unicode) + self.memcpy_addr = self.cpu.cast_ptr_to_int(support.memcpy_fn) + self._build_failure_recovery(False) + self._build_failure_recovery(True) + if self.cpu.supports_floats: + self._build_failure_recovery(False, withfloats=True) + self._build_failure_recovery(True, withfloats=True) + support.ensure_sse2_floats() + self._build_float_constants() + if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): + self._build_malloc_fixedsize_slowpath() + debug_start('jit-backend-counts') + self.set_debug(have_debug_prints()) + debug_stop('jit-backend-counts') + def setup(self): - if self.mc is None: - # the address of the function called by 'new' - gc_ll_descr = self.cpu.gc_ll_descr - gc_ll_descr.initialize() - ll_new = gc_ll_descr.get_funcptr_for_new() - self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) - if gc_ll_descr.get_funcptr_for_newarray is not None: - ll_new_array = gc_ll_descr.get_funcptr_for_newarray() - self.malloc_array_func_addr = rffi.cast(lltype.Signed, - ll_new_array) - if gc_ll_descr.get_funcptr_for_newstr is not None: - ll_new_str = gc_ll_descr.get_funcptr_for_newstr() - self.malloc_str_func_addr = rffi.cast(lltype.Signed, - ll_new_str) - if gc_ll_descr.get_funcptr_for_newunicode is not None: - ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() - self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, - ll_new_unicode) - self.memcpy_addr = self.cpu.cast_ptr_to_int(codebuf.memcpy_fn) - self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) - self._build_failure_recovery(False) - self._build_failure_recovery(True) - if self.cpu.supports_floats: - self._build_failure_recovery(False, withfloats=True) - self._build_failure_recovery(True, withfloats=True) - codebuf.ensure_sse2_floats() - self._build_float_constants() - if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): - self._build_malloc_fixedsize_slowpath() - s = os.environ.get('PYPYLOG') - if s: - if s.find(':') != -1: - s = s.split(':')[-1] - self.set_debug(True) - # Intialize here instead of __init__ to prevent - # pending_guard_tokens from being considered a prebuilt object, - # which sometimes causes memory leaks since the prebuilt list is - # still considered a GC root after we re-assign - # pending_guard_tokens in write_pending_failure_recoveries - self.pending_guard_tokens = [] + assert self.memcpy_addr != 0, "setup_once() not called?" + self.pending_guard_tokens = [] + self.mc = codebuf.MachineCodeBlockWrapper() + + def teardown(self): + self.pending_guard_tokens = None + self.mc = None + self.looppos = -1 + self.currently_compiling_loop = None def finish_once(self): if self._debug: debug_start('jit-backend-counts') for i in range(len(self.loop_run_counters)): - name, struct = self.loop_run_counters[i] - debug_print(str(name) + ':' + str(struct.i)) + struct = self.loop_run_counters[i] + debug_print(str(i) + ':' + str(struct.i)) debug_stop('jit-backend-counts') def _build_float_constants(self): @@ -268,47 +166,51 @@ def _build_malloc_fixedsize_slowpath(self): # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = self.mc.tell() + mc = codebuf.MachineCodeBlockWrapper() if self.cpu.supports_floats: # save the XMM registers in for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 - self.mc.MOVSD_sx((WORD*2)+8*i, i) - self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + mc.MOVSD_sx((WORD*2)+8*i, i) + mc.SUB_rr(edx.value, eax.value) # compute the size we want if IS_X86_32: - self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + mc.MOV_sr(WORD, edx.value) # save it as the new argument elif IS_X86_64: # rdi can be clobbered: its content was forced to the stack # by _fastpath_malloc(), like all other save_around_call_regs. - self.mc.MOV_rr(edi.value, edx.value) + mc.MOV_rr(edi.value, edx.value) addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - self.mc.JMP(imm(addr)) # tail call to the real malloc + mc.JMP(imm(addr)) # tail call to the real malloc + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.malloc_fixedsize_slowpath1 = rawstart # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = self.mc.tell() + mc = codebuf.MachineCodeBlockWrapper() if self.cpu.supports_floats: # restore the XMM registers for i in range(self.cpu.NUM_REGS):# from where they were saved - self.mc.MOVSD_xs(i, (WORD*2)+8*i) + mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - self.mc.RET() - self.mc.done() + mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + mc.RET() + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.malloc_fixedsize_slowpath2 = rawstart def assemble_loop(self, inputargs, operations, looptoken, log): - """adds the following attributes to looptoken: + '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) _x86_bootstrap_code (an integer giving an address) - _x86_direct_bootstrap_code + _x86_direct_bootstrap_code ( " " " " ) _x86_frame_depth _x86_param_depth _x86_arglocs _x86_debug_checksum - """ - looptoken.compiled_loop_token = CompiledLoopToken(self.cpu, - looptoken.number) + ''' + clt = CompiledLoopToken(self.cpu, looptoken.number) + looptoken.compiled_loop_token = clt if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) self.setup() + self.currently_compiling_loop = looptoken funcname = self._find_debug_merge_point(operations) if log: self._register_counter() @@ -318,43 +220,61 @@ arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - # profile support - name = "Loop # %s: %s" % (looptoken.number, funcname) - self.mc.start_function(name) - looptoken._x86_bootstrap_code = self.mc.tell() - adr_stackadjust = self._assemble_bootstrap_code(inputargs, arglocs) - curadr = self.mc.tell() - looptoken._x86_loop_code = curadr + bootstrappos = self.mc.get_relative_pos() + stackadjustpos = self._assemble_bootstrap_code(inputargs, arglocs) + self.looppos = self.mc.get_relative_pos() looptoken._x86_frame_depth = -1 # temporarily looptoken._x86_param_depth = -1 # temporarily frame_depth, param_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - looptoken._x86_direct_bootstrap_code = self.mc.tell() - self._assemble_bootstrap_direct_call(arglocs, curadr, + directbootstrappos = self.mc.get_relative_pos() + self._assemble_bootstrap_direct_call(arglocs, self.looppos, frame_depth+param_depth) - # - debug_print("Loop #%d has address %x to %x" % (looptoken.number, - looptoken._x86_loop_code, - self.mc.tell())) - self.mc.end_function() self.write_pending_failure_recoveries() - - def assemble_bridge(self, faildescr, inputargs, operations, log): + fullsize = self.mc.get_relative_pos() + # + rawstart = self.materialize(looptoken) + debug_print("Loop #%d (%s) has address %x to %x" % ( + looptoken.number, funcname, + rawstart + self.looppos, + rawstart + directbootstrappos)) + self._patch_stackadjust(rawstart + stackadjustpos, + frame_depth + param_depth) + self.patch_pending_failure_recoveries(rawstart) + # + looptoken._x86_bootstrap_code = rawstart + bootstrappos + looptoken._x86_loop_code = rawstart + self.looppos + looptoken._x86_direct_bootstrap_code = rawstart + directbootstrappos + self.teardown() + # oprofile support + if self.cpu.profile_agent is not None: + name = "Loop # %s: %s" % (looptoken.number, funcname) + self.cpu.profile_agent.native_code_written(name, + rawstart, fullsize) + + def assemble_bridge(self, faildescr, inputargs, operations, + original_loop_token, log): if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) + descr_number = self.cpu.get_fail_descr_number(faildescr) + try: + failure_recovery = self._find_failure_recovery_bytecode(faildescr) + except ValueError: + debug_print("Bridge out of guard", descr_number, + "was already compiled!") + return + self.setup() funcname = self._find_debug_merge_point(operations) if log: self._register_counter() operations = self._inject_debugging_code(faildescr, operations) - arglocs = self.rebuild_faillocs_from_descr( - faildescr._x86_failure_recovery_bytecode) + arglocs = self.rebuild_faillocs_from_descr(failure_recovery) if not we_are_translated(): assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) @@ -363,37 +283,58 @@ regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) - # oprofile support - descr_number = self.cpu.get_fail_descr_number(faildescr) - name = "Bridge # %s: %s" % (descr_number, funcname) - self.mc.start_function(name) - - adr_bridge = self.mc.tell() - adr_stackadjust = self._patchable_stackadjust() + stackadjustpos = self._patchable_stackadjust() frame_depth, param_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) + codeendpos = self.mc.get_relative_pos() + self.write_pending_failure_recoveries() + fullsize = self.mc.get_relative_pos() + # + rawstart = self.materialize(original_loop_token) + + debug_print("Bridge out of guard %d (%s) has address %x to %x" % + (descr_number, funcname, rawstart, rawstart + codeendpos)) + self._patch_stackadjust(rawstart + stackadjustpos, + frame_depth + param_depth) + self.patch_pending_failure_recoveries(rawstart) if not we_are_translated(): # for the benefit of tests faildescr._x86_bridge_frame_depth = frame_depth faildescr._x86_bridge_param_depth = param_depth # patch the jump from original guard - self.patch_jump_for_descr(faildescr, adr_bridge) - debug_print("Bridge out of guard %d has address %x to %x" % - (descr_number, adr_bridge, self.mc.tell())) - self.mc.end_function() - self.write_pending_failure_recoveries() + self.patch_jump_for_descr(faildescr, rawstart) + self.teardown() + # oprofile support + if self.cpu.profile_agent is not None: + name = "Bridge # %s: %s" % (descr_number, funcname) + self.cpu.profile_agent.native_code_written(name, + rawstart, fullsize) def write_pending_failure_recoveries(self): + # for each pending guard, generate the code of the recovery stub + # at the end of self.mc. for tok in self.pending_guard_tokens: - # Okay to write to _mc because we've already made sure that - # there's enough space by "reserving" bytes. - addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) - tok.faildescr._x86_adr_recovery_stub = addr - self.patch_jump_for_descr(tok.faildescr, addr) + tok.pos_recovery_stub = self.generate_quick_failure(tok) - self.pending_guard_tokens = [] - self.mc.reset_reserved_bytes() - self.mc.done() + def patch_pending_failure_recoveries(self, rawstart): + # after we wrote the assembler to raw memory, set up + # tok.faildescr._x86_adr_jump_offset to contain the raw address of + # the 4-byte target field in the JMP/Jcond instruction, and patch + # the field in question to point (initially) to the recovery stub + for tok in self.pending_guard_tokens: + addr = rawstart + tok.pos_jump_offset + tok.faildescr._x86_adr_jump_offset = addr + relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4) + assert rx86.fits_in_32bits(relative_target) + p = rffi.cast(rffi.INTP, addr) + p[0] = rffi.cast(rffi.INT, relative_target) + + def materialize(self, looptoken): + clt = looptoken.compiled_loop_token + if clt.asmmemmgr_blocks is None: + clt.asmmemmgr_blocks = [] + return self.mc.materialize(self.cpu.asmmemmgr, + clt.asmmemmgr_blocks, + self.cpu.gc_ll_descr.gcrootmap) def _find_debug_merge_point(self, operations): @@ -408,30 +349,50 @@ def _register_counter(self): if self._debug: - # YYY leak -- just put it in self.mc instead + # YYY very minor leak -- we need the counters to stay alive + # forever, just because we want to report them at the end + # of the process struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', track_allocation=False) struct.i = 0 - self.loop_run_counters.append((len(self.loop_run_counters), struct)) - + self.loop_run_counters.append(struct) + + def _find_failure_recovery_bytecode(self, faildescr): + adr_jump_offset = faildescr._x86_adr_jump_offset + if adr_jump_offset == 0: + raise ValueError + # follow the JMP/Jcond + p = rffi.cast(rffi.INTP, adr_jump_offset) + adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) + # skip the CALL + if WORD == 4: + adr_target += 5 # CALL imm + else: + adr_target += 13 # MOV r11, imm; CALL *r11 + return adr_target + def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - adr_recovery_stub = faildescr._x86_adr_recovery_stub + assert adr_jump_offset != 0 offset = adr_new_target - (adr_jump_offset + 4) # If the new target fits within a rel32 of the jump, just patch # that. Otherwise, leave the original rel32 to the recovery stub in # place, but clobber the recovery stub with a jump to the real # target. + mc = codebuf.MachineCodeBlockWrapper() if rx86.fits_in_32bits(offset): - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) mc.writeimm32(offset) + mc.copy_to_raw_memory(adr_jump_offset) else: - # "mov r11, addr; jmp r11" is 13 bytes - mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + # "mov r11, addr; jmp r11" is 13 bytes, which fits in there + # because we always write "mov r11, addr; call *r11" in the + # first place. mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) mc.JMP_r(X86_64_SCRATCH_REG.value) - - mc.done() + p = rffi.cast(rffi.INTP, adr_jump_offset) + adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) + mc.copy_to_raw_memory(adr_target) + faildescr._x86_adr_jump_offset = 0 # means "patched" @specialize.argtype(1) def _inject_debugging_code(self, looptoken, operations): @@ -442,7 +403,7 @@ s += op.getopnum() looptoken._x86_debug_checksum = s c_adr = ConstInt(rffi.cast(lltype.Signed, - self.loop_run_counters[-1][1])) + self.loop_run_counters[-1])) box = BoxInt() box2 = BoxInt() ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], @@ -451,19 +412,11 @@ ResOperation(rop.SETFIELD_RAW, [c_adr, box2], None, descr=self.debug_counter_descr)] operations = ops + operations - # # we need one register free (a bit of a hack, but whatever) - # self.mc.PUSH(eax) - # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) - # self.mc.MOV(eax, heap(adr)) - # self.mc.ADD(eax, imm1) - # self.mc.MOV(heap(adr), eax) - # self.mc.POP(eax) return operations def _assemble(self, regalloc, operations): self._regalloc = regalloc regalloc.walk_operations(operations) - self.mc.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -479,30 +432,31 @@ def _patchable_stackadjust(self): # stack adjustment LEA self.mc.LEA32_rb(esp.value, 0) - return self.mc.tell() - 4 + return self.mc.get_relative_pos() - 4 - def _patch_stackadjust(self, adr_lea, reserved_depth): + def _patch_stackadjust(self, adr_lea, allocated_depth): # patch stack adjustment LEA - mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) - # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. + mc = codebuf.MachineCodeBlockWrapper() + # Compute the correct offset for the instruction LEA ESP, [EBP-4*words] + mc.writeimm32(self._get_offset_of_ebp_from_esp(allocated_depth)) + mc.copy_to_raw_memory(adr_lea) + + def _get_offset_of_ebp_from_esp(self, allocated_depth): # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth - # align, e.g. for Mac OS X + words = (self.cpu.FRAME_FIXED_SIZE - 1) + allocated_depth + # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.writeimm32(-WORD * aligned_words) - mc.done() + return -WORD * aligned_words def _call_header(self): + # NB. the shape of the frame is hard-coded in get_basic_shape() too. + # Also, make sure this is consistent with FRAME_FIXED_SIZE. self.mc.PUSH_r(ebp.value) self.mc.MOV_rr(ebp.value, esp.value) for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) - # NB. the shape of the frame is hard-coded in get_basic_shape() too. - # Also, make sure this is consistent with FRAME_FIXED_SIZE. - return self._patchable_stackadjust() - def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -512,17 +466,16 @@ self.mc.POP_r(ebp.value) self.mc.RET() - def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + def _assemble_bootstrap_direct_call(self, arglocs, jmppos, stackdepth): if IS_X86_64: - return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) + return self._assemble_bootstrap_direct_call_64(arglocs, jmppos, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - # XXX not to repeat the logic, a bit around - adr_stackadjust = self._call_header() - self._patch_stackadjust(adr_stackadjust, stackdepth) + self._call_header() + self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if isinstance(loc, RegLoc): @@ -544,9 +497,11 @@ self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) assert isinstance(loc, StackLoc) self.mc.MOVSD_bx(loc.value, xmmtmp.value) - self.mc.JMP_l(jmpadr) + endpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(jmppos - endpos) + assert endpos == self.mc.get_relative_pos() - def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + def _assemble_bootstrap_direct_call_64(self, arglocs, jmppos, stackdepth): # XXX: Very similar to _emit_call_64 src_locs = [] @@ -560,8 +515,8 @@ unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - adr_stackadjust = self._call_header() - self._patch_stackadjust(adr_stackadjust, stackdepth) + self._call_header() + self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones assert len(nonfloatlocs) == len(floatlocs) @@ -597,8 +552,9 @@ # clobber the scratch register self.mc.MOV(loc, X86_64_SCRATCH_REG) - finaljmp = self.mc.tell() - self.mc.JMP(imm(jmpadr)) + endpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(jmppos - endpos) + assert endpos == self.mc.get_relative_pos() def redirect_call_assembler(self, oldlooptoken, newlooptoken): # some minimal sanity checking @@ -611,16 +567,17 @@ # Ideally we should rather patch all existing CALLs, but well. oldadr = oldlooptoken._x86_direct_bootstrap_code target = newlooptoken._x86_direct_bootstrap_code - mc = codebuf.InMemoryCodeBuilder(oldadr, oldadr + 16) + mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) - mc.done() + mc.copy_to_raw_memory(oldadr) def _assemble_bootstrap_code(self, inputargs, arglocs): nonfloatlocs, floatlocs = arglocs - adr_stackadjust = self._call_header() + self._call_header() + stackadjustpos = self._patchable_stackadjust() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] - self.mc._mc.begin_reuse_scratch_register() + self.mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: @@ -652,8 +609,8 @@ self.mc.MOVSD(xmmtmp, heap(adr)) assert isinstance(loc, StackLoc) self.mc.MOVSD_bx(loc.value, xmmtmp.value) - self.mc._mc.end_reuse_scratch_register() - return adr_stackadjust + self.mc.end_reuse_scratch_register() + return stackadjustpos def dump(self, text): if not self.verbose: @@ -661,7 +618,8 @@ _prev = Box._extended_display try: Box._extended_display = False - print >> sys.stderr, ' 0x%x %s' % (fixid(self.mc.tell()), text) + pos = self.mc.get_relative_pos() + print >> sys.stderr, ' 0x%x %s' % (pos, text) finally: Box._extended_display = _prev @@ -721,7 +679,7 @@ arglocs, resloc) if not we_are_translated(): # must be added by the genop_guard_list[]() - assert hasattr(faildescr, '_x86_adr_jump_offset') + assert guard_token is self.pending_guard_tokens[-1] def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc, current_depths): @@ -794,9 +752,6 @@ result_loc): guard_opnum = guard_op.getopnum() self.mc.UCOMISD(arglocs[0], arglocs[1]) - # 16 is enough space for the rel8 jumps below and the rel32 - # jump in implement_guard - self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: if need_jp: self.mc.J_il8(rx86.Conditions['P'], 6) @@ -964,9 +919,6 @@ def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.getopnum() self.mc.UCOMISD(arglocs[0], arglocs[1]) - # 16 is enough space for the rel8 jumps below and the rel32 - # jump in implement_guard - self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: self.mc.J_il8(rx86.Conditions['P'], 6) self.implement_guard(guard_token, 'E') @@ -1290,13 +1242,11 @@ self.mc.CMP32_mi((locs[0].value, 0), expected_typeid) def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): - self.mc.ensure_bytes_available(256) self._cmp_guard_class(locs) self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, guard_token, locs, ign_2): - self.mc.ensure_bytes_available(256) self.mc.CMP(locs[0], imm1) # Patched below self.mc.J_il8(rx86.Conditions['B'], 0) @@ -1305,7 +1255,7 @@ # patch the JB above offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, [chr(offset)]) + self.mc.overwrite(jb_location-1, chr(offset)) # self.implement_guard(guard_token, 'NE') @@ -1314,42 +1264,33 @@ exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - desc_bytes = self.failure_recovery_description(failargs, fail_locs) - return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) + return GuardToken(faildescr, failargs, fail_locs, exc) - def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): + def generate_quick_failure(self, guardtok): """Generate the initial code for handling a failure. We try to - keep it as compact as possible. The idea is that this code is - executed at most once (and very often, zero times); when - executed, it generates a more complete piece of code which can - really handle recovery from this particular failure. + keep it as compact as possible. """ - fail_index = self.cpu.get_fail_descr_number(faildescr) - addr = mc.tell() + fail_index = self.cpu.get_fail_descr_number(guardtok.faildescr) + mc = self.mc + startpos = mc.get_relative_pos() withfloats = False - for box in failargs: + for box in guardtok.failargs: if box is not None and box.type == FLOAT: withfloats = True break + exc = guardtok.exc mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery - faildescr._x86_failure_recovery_bytecode = mc.tell() - for byte in desc_bytes: - mc.writechr(ord(byte)) + self.write_failure_recovery_description(mc, guardtok.failargs, + guardtok.fail_locs) # write the fail_index too mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): - mc.writechr(0xCC) - faildescr._x86_debug_faillocs = [loc for loc in fail_locs - if loc is not None] - - # Make sure the recovery stub is at least 16 bytes long (for the - # case where we overwrite the recovery stub with a 64-bit absolute - # jump) - while mc.tell() - addr < 16: - mc.writechr(0x00) - return addr + mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.fail_locs if loc is not None] + guardtok.faildescr._x86_debug_faillocs = faillocs + return startpos DESCR_REF = 0x00 DESCR_INT = 0x01 @@ -1360,8 +1301,7 @@ CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def failure_recovery_description(self, failargs, locs): - desc_bytes = [] + def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1381,19 +1321,14 @@ n = loc.value n = kind + 4*n while n > 0x7F: - desc_bytes.append(chr((n & 0x7F) | 0x80)) + mc.writechar(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - desc_bytes.append(chr(n)) - desc_bytes.append(chr(self.CODE_STOP)) + mc.writechar(chr(n)) + mc.writechar(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE - return desc_bytes - - def write_failure_recovery_description(self, mc, failargs, locs): - for byte in self.failure_recovery_description(failargs, locs): - mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager @@ -1531,10 +1466,8 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc._mc - # Assume that we are called at the beginning, when there is no risk - # that 'mc' runs out of space. Checked by asserts in mc.write(). - recovery_addr = mc.tell() + mc = codebuf.MachineCodeBlockWrapper() + self.mc = mc # Push all general purpose registers for gpr in range(self.cpu.NUM_REGS-1, -1, -1): @@ -1585,11 +1518,12 @@ # above. self._call_footer() - self.mc.done() - self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None def generate_failure(self, fail_index, locs, exc, locs_are_ref): - self.mc._mc.begin_reuse_scratch_register() + self.mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] if isinstance(loc, RegLoc): @@ -1616,7 +1550,7 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(eax, loc) self.mc.MOV(heap(adr), eax) - self.mc._mc.end_reuse_scratch_register() + self.mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1632,17 +1566,13 @@ self._call_footer() def implement_guard(self, guard_token, condition=None): - self.mc.reserve_bytes(guard_token.recovery_stub_size()) - self.pending_guard_tokens.append(guard_token) - # These jumps are patched later, the mc.tell() are just - # dummy values. Also, use self.mc._mc to avoid triggering a - # "buffer full" exactly here. - mc = self.mc._mc + # These jumps are patched later. if condition: - mc.J_il(rx86.Conditions[condition], mc.tell()) + self.mc.J_il(rx86.Conditions[condition], 0) else: - mc.JMP_l(mc.tell()) - guard_token.faildescr._x86_adr_jump_offset = mc.tell() - 4 + self.mc.JMP_l(0) + guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4 + self.pending_guard_tokens.append(guard_token) def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] @@ -1697,7 +1627,6 @@ # Write a call to the direct_bootstrap_code of the target assembler self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1733,7 +1662,7 @@ # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - self.mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, chr(offset)) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1768,7 +1697,7 @@ # Here we join Path A and Path B again offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.overwrite(jmp_location - 1, chr(offset)) self.mc.CMP_bi(FORCE_INDEX_OFS, 0) self.implement_guard(guard_token, 'L') @@ -1783,9 +1712,6 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - # ensure that enough bytes are available to write the whole - # following piece of code atomically (for the JZ) - self.mc.ensure_bytes_available(256) self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), descr.jit_wb_if_flag_singlebyte) self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later @@ -1828,7 +1754,7 @@ # patch the JZ above offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - self.mc.overwrite(jz_location-1, [chr(offset)]) + self.mc.overwrite(jz_location-1, chr(offset)) def genop_force_token(self, op, arglocs, resloc): # RegAlloc.consider_force_token ensures this: @@ -1851,18 +1777,21 @@ gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: mark = self._regalloc.get_mark_gc_roots(gcrootmap) - gcrootmap.put(rffi.cast(llmemory.Address, self.mc.tell()), mark) + self.mc.insert_gcroot_marker(mark) def target_arglocs(self, loop_token): return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(imm(loop_token._x86_loop_code)) + if loop_token is self.currently_compiling_loop: + curpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(self.looppos - curpos) + else: + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): size = max(size, self.cpu.gc_ll_descr.minimal_size_in_nursery) - self.mc.ensure_bytes_available(256) self.mc.MOV(eax, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (eax.value, size)) self.mc.CMP(edx, heap(nursery_top_adr)) @@ -1890,7 +1819,7 @@ offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.overwrite(jmp_adr-1, chr(offset)) # on 64-bits, 'tid' is a value that fits in 31 bits self.mc.MOV_mi((eax.value, 0), tid) self.mc.MOV(heap(nursery_free_adr), edx) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py Wed Dec 1 18:08:21 2010 @@ -320,11 +320,22 @@ def locs_for_fail(self, guard_op): return [self.loc(v) for v in guard_op.getfailargs()] + def get_current_depth(self): + # return (self.fm.frame_depth, self.param_depth), but trying to share + # the resulting tuple among several calls + arg0 = self.fm.frame_depth + arg1 = self.param_depth + result = self.assembler._current_depths_cache + if result[0] != arg0 or result[1] != arg1: + result = (arg0, arg1) + self.assembler._current_depths_cache = result + return result + def perform_with_guard(self, op, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) self.rm.position += 1 self.xrm.position += 1 - current_depths = (self.fm.frame_depth, self.param_depth) + current_depths = self.get_current_depth() self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, current_depths) @@ -340,7 +351,7 @@ arglocs)) else: self.assembler.dump('%s(%s)' % (guard_op, arglocs)) - current_depths = (self.fm.frame_depth, self.param_depth) + current_depths = self.get_current_depth() self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, current_depths) @@ -1097,7 +1108,7 @@ if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert reg in self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX gcrootmap.add_callee_save_reg(shape, self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX[reg]) - return gcrootmap.compress_callshape(shape) + return shape def consider_force_token(self, op): loc = self.rm.force_allocate_reg(op.result) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/regloc.py Wed Dec 1 18:08:21 2010 @@ -334,9 +334,9 @@ if code == possible_code: val = getattr(loc, "value_" + possible_code)() if possible_code == 'i': - offset = intmask(val - (self.tell() + 5)) - if rx86.fits_in_32bits(offset): + if self.WORD == 4: _rx86_getattr(self, name + "_l")(val) + self.add_pending_relocation() else: assert self.WORD == 8 self._load_scratch(val) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/runner.py Wed Dec 1 18:08:21 2010 @@ -44,6 +44,7 @@ def setup_once(self): self.profile_agent.startup() + self.assembler.setup_once() def finish_once(self): self.assembler.finish_once() @@ -58,7 +59,7 @@ clt = original_loop_token.compiled_loop_token clt.compiling_a_bridge() self.assembler.assemble_bridge(faildescr, inputargs, operations, - log=log) + original_loop_token, log=log) def set_future_value_int(self, index, intvalue): self.assembler.fail_boxes_int.setitem(index, intvalue) @@ -127,8 +128,8 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = -1 - bytecode = rffi.cast(rffi.UCHARP, - faildescr._x86_failure_recovery_bytecode) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) # start of "no gc operation!" block fail_index_2 = self.assembler.grab_frame_values( bytecode, @@ -168,17 +169,12 @@ CPU = CPU386 # silence warnings - -history.LoopToken._x86_param_depth = 0 -history.LoopToken._x86_arglocs = (None, None) -history.LoopToken._x86_frame_depth = 0 -history.LoopToken._x86_bootstrap_code = 0 -history.LoopToken._x86_direct_bootstrap_code = 0 -history.LoopToken._x86_failure_recovery_bytecode = 0 -history.LoopToken._x86_loop_code = 0 -history.LoopToken._x86_current_depths = (0, 0) - -compile._DoneWithThisFrameDescr._x86_current_depths = (0, 0) -compile._DoneWithThisFrameDescr._x86_failure_recovery_bytecode = 0 -compile._DoneWithThisFrameDescr._x86_adr_jump_offset = 0 - +##history.LoopToken._x86_param_depth = 0 +##history.LoopToken._x86_arglocs = (None, None) +##history.LoopToken._x86_frame_depth = 0 +##history.LoopToken._x86_bootstrap_code = 0 +##history.LoopToken._x86_direct_bootstrap_code = 0 +##history.LoopToken._x86_loop_code = 0 +##history.LoopToken._x86_debug_checksum = 0 +##compile.AbstractFailDescr._x86_current_depths = (0, 0) +##compile.AbstractFailDescr._x86_adr_jump_offset = 0 Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/rx86.py Wed Dec 1 18:08:21 2010 @@ -137,10 +137,9 @@ # ____________________________________________________________ # Emit an immediate displacement (relative to the cur insn) -def encode_relative(mc, target, _, orbyte): +def encode_relative(mc, relative_target, _, orbyte): assert orbyte == 0 - offset = intmask(target - (mc.tell() + 4)) - mc.writeimm32(offset) + mc.writeimm32(relative_target) return 0 def relative(argnum): Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/support.py Wed Dec 1 18:08:21 2010 @@ -1,4 +1,7 @@ +import sys from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from pypy.translator.tool.cbuild import ExternalCompilationInfo + def values_array(TP, size): ATP = lltype.GcArray(TP) @@ -23,3 +26,22 @@ return True return ValuesArray() + +# ____________________________________________________________ + +memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) + +# ____________________________________________________________ + +if sys.platform == 'win32': + ensure_sse2_floats = lambda : None +else: + _sse2_eci = ExternalCompilationInfo( + compile_extra = ['-msse2', '-mfpmath=sse'], + separate_module_sources = ['void PYPY_NO_OP(void) {}'], + ) + ensure_sse2_floats = rffi.llexternal('PYPY_NO_OP', [], lltype.Void, + compilation_info=_sse2_eci, + sandboxsafe=True) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_assembler.py Wed Dec 1 18:08:21 2010 @@ -1,5 +1,5 @@ from pypy.jit.backend.x86.regloc import * -from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper +from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask @@ -19,28 +19,10 @@ return 42 class FakeMC: - def __init__(self, base_address=0): + def __init__(self): self.content = [] - self._size = 100 - self.base_address = base_address - def writechr(self, n): - self.content.append(n) - def tell(self): - return self.base_address + len(self.content) - def get_relative_pos(self): - return len(self.content) - def JMP(self, *args): - self.content.append(("JMP", args)) - def done(self): - pass - def PUSH_r(self, reg): - pass - def POP_r(self, reg): - pass - -class FakeAssembler: - def write_pending_failure_recoveries(self): - pass + def writechar(self, char): + self.content.append(ord(char)) def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -255,41 +237,3 @@ assert assembler.fail_boxes_int.getitem(i) == expected_ints[i] assert assembler.fail_boxes_ptr.getitem(i) == expected_ptrs[i] assert assembler.fail_boxes_float.getitem(i) == expected_floats[i] - -class FakeProfileAgent(object): - def __init__(self): - self.functions = [] - - def native_code_written(self, name, address, size): - self.functions.append((name, address, size)) - -class FakeMCWrapper(MachineCodeBlockWrapper): - count = 0 - def _instantiate_mc(self): - self.count += 1 - return FakeMC(200 * (self.count - 1)) - -def test_mc_wrapper_profile_agent(): - agent = FakeProfileAgent() - assembler = FakeAssembler() - mc = FakeMCWrapper(assembler, 100, agent) - mc.start_function("abc") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4)] - mc.writechr("x") - mc.start_function("cde") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4), ("cde", 5, 4)] - mc.start_function("xyz") - for i in range(50): - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4), ("cde", 5, 4), ("xyz", 9, 29), ("xyz", 200, 22)] Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_gc_integration.py Wed Dec 1 18:08:21 2010 @@ -33,9 +33,6 @@ def add_callee_save_reg(self, shape, reg_index): index_to_name = { 1: 'ebx', 2: 'esi', 3: 'edi' } shape.append(index_to_name[reg_index]) - def compress_callshape(self, shape): - assert shape[0] == 'shape' - return ['compressed'] + shape[1:] class MockGcDescr(GcCache): def get_funcptr_for_new(self): @@ -58,6 +55,7 @@ def test_mark_gc_roots(self): cpu = CPU(None, None) + cpu.setup_once() regalloc = RegAlloc(MockAssembler(cpu, MockGcDescr(False))) boxes = [BoxPtr() for i in range(len(X86RegisterManager.all_regs))] longevity = {} @@ -81,7 +79,7 @@ assert len(regalloc.assembler.movs) == 3 # mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) - assert mark[0] == 'compressed' + assert mark[0] == 'shape' base = -WORD * FRAME_FIXED_SIZE expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) @@ -90,6 +88,7 @@ cpu = CPU(None, None) cpu.gc_ll_descr = MockGcDescr(False) + cpu.setup_once() S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', ('field', lltype.Ptr(S)), @@ -214,6 +213,7 @@ cpu = CPU(None, None) cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() + cpu.setup_once() NODE = lltype.Struct('node', ('tid', lltype.Signed), ('value', lltype.Signed)) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc.py Wed Dec 1 18:08:21 2010 @@ -41,7 +41,10 @@ self.movs = [] self.performs = [] self.lea = [] - self.cpu = cpu or CPU(None, None) + if cpu is None: + cpu = CPU(None, None) + cpu.setup_once() + self.cpu = cpu if gc_ll_descr is None: gc_ll_descr = MockGcDescr(False) self.cpu.gc_ll_descr = gc_ll_descr @@ -76,6 +79,7 @@ class BaseTestRegalloc(object): cpu = CPU(None, None) + cpu.setup_once() def raising_func(i): if i: Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc2.py Wed Dec 1 18:08:21 2010 @@ -19,6 +19,7 @@ ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, 9) @@ -41,6 +42,7 @@ ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, -10) @@ -137,6 +139,7 @@ ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, -13) @@ -251,6 +254,7 @@ ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, 17) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regloc.py Wed Dec 1 18:08:21 2010 @@ -1,4 +1,4 @@ -import struct +import struct, sys from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.test.test_rx86 import CodeBuilder32, CodeBuilder64, assert_encodes_as from pypy.jit.backend.x86.assembler import heap @@ -37,26 +37,36 @@ assert_encodes_as(cb64, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') assert_encodes_as(cb64, "CMP16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\x81\x7D\x00\x39\x30') -def test_jmp_wraparound(): - if not IS_X86_32: - py.test.skip() - - pos_addr = intmask(0x7FFFFF00) - neg_addr = intmask(0x800000BB) - - # JMP to "negative" address from "positive" address - s = cb32() - s.base_address = pos_addr - s.JMP(ImmedLoc(neg_addr)) - expected_ofs = neg_addr - (pos_addr+5) - assert s.getvalue() == '\xE9' + struct.pack(" sys.maxint: + continue + mc = codebuf.MachineCodeBlockWrapper() + mc.CALL(ImmedLoc(target)) + length = mc.get_relative_pos() + buf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw') + rawstart = rffi.cast(lltype.Signed, buf) + if IS_X86_32: + assert length == 5 + assert mc.relocations == [5] + expected = "\xE8" + struct.pack(' 5: + x = A1 + else: + x = A2 + x() + rtyper = support.annotate(f, [35]) + maingraph = rtyper.annotator.translator.graphs[0] + cw = CodeWriter(FakeCPU(rtyper), [FakeJitDriverSD(maingraph)]) + cw.find_all_graphs(MyFakePolicy()) + cw.make_jitcodes(verbose=True) + # + names = [jitcode.name for jitcode in cw.assembler.indirectcalltargets] + assert len(names) == 1 + assert names[0].startswith('instantiate_') and names[0].endswith('A1') + # + print cw.assembler.list_of_addr2name + names = dict.fromkeys([value + for key, value in cw.assembler.list_of_addr2name]) + assert 'A1' in names + assert 'A2' in names + def test_int_abs(): def f(n): return abs(n) Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_list.py Wed Dec 1 18:08:21 2010 @@ -56,9 +56,8 @@ if '/' in oopspec_name: oopspec_name, property = oopspec_name.split('/') def force_flags(op): - if property == 'NONNEG': return True, False - if property == 'NEG': return False, False - if property == 'CANRAISE': return False, True + if property == 'NONNEG': return True + if property == 'NEG': return False raise ValueError(property) tr._get_list_nonneg_canraise_flags = force_flags op = SpaceOperation('direct_call', @@ -122,9 +121,6 @@ check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_i %r0, , %i1 -> %i2 """) - builtin_test('list.getitem/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -139,9 +135,6 @@ check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_pure_i %r0, , %i1 -> %i2 """) - builtin_test('list.getitem_foldable/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -158,10 +151,6 @@ check_neg_index %r0, , %i0 -> %i1 setarrayitem_gc_i %r0, , %i1, %i2 """) - builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), - varoftype(lltype.Signed), - varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -206,9 +195,6 @@ check_resizable_neg_index %r0, , %i0 -> %i1 getlistitem_gc_i %r0, , , %i1 -> %i2 """) - builtin_test('list.getitem/CANRAISE', - [varoftype(VARLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_resizable_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), @@ -225,10 +211,6 @@ check_resizable_neg_index %r0, , %i0 -> %i1 setlistitem_gc_i %r0, , , %i1, %i2 """) - builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), - varoftype(lltype.Signed), - varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_resizable_len(): builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_void_list.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_void_list.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_void_list.py Wed Dec 1 18:08:21 2010 @@ -51,9 +51,6 @@ builtin_test('list.getitem/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -62,9 +59,6 @@ builtin_test('list.getitem_foldable/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem_foldable/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -75,10 +69,6 @@ varoftype(lltype.Signed), varoftype(lltype.Void)], lltype.Void, "") - builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), - varoftype(lltype.Signed), - varoftype(lltype.Void)], - lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -115,9 +105,6 @@ builtin_test('list.getitem/NEG', [varoftype(VARLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem/CANRAISE', - [varoftype(VARLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_resizable_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), @@ -128,10 +115,6 @@ varoftype(lltype.Signed), varoftype(lltype.Void)], lltype.Void, "") - builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), - varoftype(lltype.Signed), - varoftype(lltype.Void)], - lltype.Void, NotSupported) def test_resizable_len(): builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, Modified: pypy/branch/fast-forward/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/history.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/history.py Wed Dec 1 18:08:21 2010 @@ -4,8 +4,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic from pypy.rlib.objectmodel import compute_hash, compute_unique_id -from pypy.rlib.rarithmetic import intmask, r_longlong -from pypy.tool.uid import uid +from pypy.rlib.rarithmetic import intmask, r_int64 from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -731,7 +730,7 @@ # specnodes = ... # and more data specified by the backend when the loop is compiled number = -1 - generation = r_longlong(0) + generation = r_int64(0) # one purpose of LoopToken is to keep alive the CompiledLoopToken # returned by the backend. When the LoopToken goes away, the # CompiledLoopToken has its __del__ called, which frees the assembler Modified: pypy/branch/fast-forward/pypy/jit/metainterp/memmgr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/memmgr.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/memmgr.py Wed Dec 1 18:08:21 2010 @@ -1,5 +1,5 @@ import math -from pypy.rlib.rarithmetic import r_longlong +from pypy.rlib.rarithmetic import r_int64 from pypy.rlib.debug import debug_start, debug_print, debug_stop from pypy.rlib.objectmodel import we_are_translated @@ -24,7 +24,7 @@ def __init__(self): self.check_frequency = -1 - # NB. use of r_longlong to be extremely far on the safe side: + # NB. use of r_int64 to be extremely far on the safe side: # this is increasing by one after each loop or bridge is # compiled, and it must not overflow. If the backend implements # complete freeing in cpu.free_loop_and_bridges(), then it may @@ -32,13 +32,13 @@ # enough. But in this day and age, you'd still never have the # patience of waiting for a slowly-increasing 64-bit number to # overflow :-) - self.current_generation = r_longlong(1) - self.next_check = r_longlong(-1) + self.current_generation = r_int64(1) + self.next_check = r_int64(-1) self.alive_loops = {} def set_max_age(self, max_age, check_frequency=0): if max_age <= 0: - self.next_check = r_longlong(-1) + self.next_check = r_int64(-1) else: self.max_age = max_age if check_frequency <= 0: Modified: pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py Wed Dec 1 18:08:21 2010 @@ -612,8 +612,10 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) arrayindex = vinfo.array_field_by_descrs[arrayfielddescr] index = indexbox.getint() - if index < 0: - index += vinfo.get_array_length(virtualizable, arrayindex) + # Support for negative index: disabled + # (see codewriter/jtransform.py, _check_no_vable_array). + #if index < 0: + # index += vinfo.get_array_length(virtualizable, arrayindex) assert 0 <= index < vinfo.get_array_length(virtualizable, arrayindex) return vinfo.get_index_in_array(virtualizable, arrayindex, index) Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_basic.py Wed Dec 1 18:08:21 2010 @@ -1701,6 +1701,29 @@ res = self.meta_interp(f, [5, 2]) assert 4 < res < 14 + def test_compute_identity_hash(self): + from pypy.rlib.objectmodel import compute_identity_hash + class A(object): + pass + def f(): + a = A() + return compute_identity_hash(a) == compute_identity_hash(a) + res = self.interp_operations(f, []) + assert res + # a "did not crash" kind of test + + def test_compute_unique_id(self): + from pypy.rlib.objectmodel import compute_unique_id + class A(object): + pass + def f(): + a1 = A() + a2 = A() + return (compute_unique_id(a1) == compute_unique_id(a1) and + compute_unique_id(a1) != compute_unique_id(a2)) + res = self.interp_operations(f, []) + assert res + class TestOOtype(BasicTests, OOJitMixin): Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_list.py Wed Dec 1 18:08:21 2010 @@ -188,6 +188,26 @@ assert res == f(4) self.check_loops(call=0, getfield_gc=0) + def test_fold_indexerror(self): + jitdriver = JitDriver(greens = [], reds = ['total', 'n', 'lst']) + def f(n): + lst = [] + total = 0 + while n > 0: + jitdriver.can_enter_jit(lst=lst, n=n, total=total) + jitdriver.jit_merge_point(lst=lst, n=n, total=total) + lst.append(n) + try: + total += lst[n] + except IndexError: + total += 1000 + n -= 1 + return total + + res = self.meta_interp(f, [15], listops=True) + assert res == f(15) + self.check_loops(guard_exception=0) + class TestOOtype(ListTests, OOJitMixin): pass Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_memmgr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_memmgr.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_memmgr.py Wed Dec 1 18:08:21 2010 @@ -1,9 +1,10 @@ -import sys, py +import sys if len(sys.argv) >= 4 and sys.argv[1] == '--sub': sys.path[:] = eval(sys.argv[2]) # hacks for test_integration from pypy.conftest import option option.__dict__.update(eval(sys.argv[3])) +import py from pypy.jit.metainterp.memmgr import MemoryManager from pypy.jit.metainterp.test.test_basic import LLJitMixin from pypy.rlib.jit import JitDriver, dont_look_inside Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_recursive.py Wed Dec 1 18:08:21 2010 @@ -927,12 +927,16 @@ x=x) frame.s = hint(frame.s, promote=True) n -= 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] frame.s += 1 if codeno == 0: subframe = Frame([n, n+1, n+2, n+3], 0) x += f(1, 10, 1, subframe) - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] x += len(frame.l) frame.s -= 1 return x Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_virtualizable.py Wed Dec 1 18:08:21 2010 @@ -480,9 +480,13 @@ myjitdriver.jit_merge_point(frame=frame, n=n, x=x) frame.s = hint(frame.s, promote=True) n -= 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] frame.s += 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] x += len(frame.l) frame.s -= 1 return x @@ -994,7 +998,9 @@ jitdriver.can_enter_jit(frame=frame, n=n) jitdriver.jit_merge_point(frame=frame, n=n) popped = frame.stack[frame.stackpos] - frame.stackpos -= 1 + sp = frame.stackpos - 1 + assert sp >= 0 + frame.stackpos = sp to_push = intmask(popped * 3) frame.stack[frame.stackpos] = to_push frame.stackpos += 1 Modified: pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py Wed Dec 1 18:08:21 2010 @@ -159,6 +159,8 @@ self.check_access_directly_sanity(graphs) if backendopt: self.prejit_optimizations(policy, graphs) + elif self.opt.listops: + self.prejit_optimizations_minimal_inline(policy, graphs) self.build_meta_interp(ProfilerClass) self.make_args_specifications() @@ -256,6 +258,10 @@ remove_asserts=True, really_remove_asserts=True) + def prejit_optimizations_minimal_inline(self, policy, graphs): + from pypy.translator.backendopt.inline import auto_inline_graphs + auto_inline_graphs(self.translator, graphs, 0.01) + def build_cpu(self, CPUClass, translate_support_code=False, no_stats=False, **kwds): assert CPUClass is not None Modified: pypy/branch/fast-forward/pypy/jit/tl/spli/interpreter.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/tl/spli/interpreter.py (original) +++ pypy/branch/fast-forward/pypy/jit/tl/spli/interpreter.py Wed Dec 1 18:08:21 2010 @@ -105,26 +105,32 @@ self.stack_depth += 1 def pop(self): - self.stack_depth -= 1 - val = self.value_stack[self.stack_depth] - self.value_stack[self.stack_depth] = None + sd = self.stack_depth - 1 + assert sd >= 0 + self.stack_depth = sd + val = self.value_stack[sd] + self.value_stack[sd] = None return val def pop_many(self, n): return [self.pop() for i in range(n)] def peek(self): - return self.value_stack[self.stack_depth - 1] + sd = self.stack_depth - 1 + assert sd >= 0 + return self.value_stack[sd] def POP_TOP(self, _, next_instr, code): self.pop() return next_instr def LOAD_FAST(self, name_index, next_instr, code): + assert name_index >= 0 self.push(self.locals[name_index]) return next_instr def STORE_FAST(self, name_index, next_instr, code): + assert name_index >= 0 self.locals[name_index] = self.pop() return next_instr Modified: pypy/branch/fast-forward/pypy/jit/tl/tl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/tl/tl.py (original) +++ pypy/branch/fast-forward/pypy/jit/tl/tl.py Wed Dec 1 18:08:21 2010 @@ -16,32 +16,40 @@ def __init__(self, size): self = hint(self, access_directly=True, fresh_virtualizable=True) self.stack = [0] * size - self.stackpos = 0 + self.stackpos = 0 # always store a known-nonneg integer here def append(self, elem): self.stack[self.stackpos] = elem self.stackpos += 1 def pop(self): - self.stackpos -= 1 - if self.stackpos < 0: + stackpos = self.stackpos - 1 + if stackpos < 0: raise IndexError - return self.stack[self.stackpos] + self.stackpos = stackpos # always store a known-nonneg integer here + return self.stack[stackpos] def pick(self, i): - self.append(self.stack[self.stackpos - i - 1]) + n = self.stackpos - i - 1 + assert n >= 0 + self.append(self.stack[n]) def put(self, i): elem = self.pop() - self.stack[self.stackpos - i - 1] = elem + n = self.stackpos - i - 1 + assert n >= 0 + self.stack[n] = elem def roll(self, r): if r < -1: i = self.stackpos + r if i < 0: raise IndexError - elem = self.stack[self.stackpos - 1] + n = self.stackpos - 1 + assert n >= 0 + elem = self.stack[n] for j in range(self.stackpos - 2, i - 1, -1): + assert j >= 0 self.stack[j + 1] = self.stack[j] self.stack[i] = elem elif r > 1: @@ -51,7 +59,9 @@ elem = self.stack[i] for j in range(i, self.stackpos - 1): self.stack[j] = self.stack[j + 1] - self.stack[self.stackpos - 1] = elem + n = self.stackpos - 1 + assert n >= 0 + self.stack[n] = elem def make_interp(supports_call): Modified: pypy/branch/fast-forward/pypy/jit/tool/log-template.gnumeric ============================================================================== Binary files. No diff available. Modified: pypy/branch/fast-forward/pypy/jit/tool/log2gnumeric.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/tool/log2gnumeric.py (original) +++ pypy/branch/fast-forward/pypy/jit/tool/log2gnumeric.py Wed Dec 1 18:08:21 2010 @@ -14,25 +14,34 @@ import re, sys import gzip +import optparse -def main(): +def main(logname, options): logname = sys.argv[1] outname = logname + '.gnumeric' data = open(logname).read() data = data.replace('\n', '') - maxclock = find_max_clock(data) + minclock, maxclock = get_clock_range(data) + time0 = minclock # we want "relative clocks" + maxtime = maxclock-time0 # xml = gzip.open('log-template.gnumeric').read() - xml = replace_sheet(xml, 'translation-task', tasks_rows(data)) - xml = replace_sheet(xml, 'gc-collect', gc_collect_rows(data)) - xml = replace_sheet(xml, 'memusage', memusage_rows(logname + '.memusage', maxclock)) + xml = replace_sheet(xml, 'translation-task', tasks_rows(time0, data)) + xml = replace_sheet(xml, 'gc-collect', gc_collect_rows(time0, data)) + xml = replace_sheet(xml, 'loops', loops_rows(time0, data)) + xml = replace_sheet(xml, 'vmrss', vmrss_rows(logname + '.vmrss', maxtime)) + xml = replace_sheet(xml, 'cpython-vmrss', vmrss_rows(options.cpython_vmrss, maxtime)) # out = gzip.open(outname, 'wb') out.write(xml) out.close() +# ======================================================================== +# functions to manipulate gnumeric files +# ======================================================================== + def replace_sheet(xml, sheet_name, data): pattern = '%s.*?(.*?)' regex = re.compile(pattern % sheet_name, re.DOTALL) @@ -68,7 +77,22 @@ return '\n'.join(parts) -def gc_collect_rows(data): +# ======================================================================== +# functions to extract various data from the logs +# ======================================================================== + +CLOCK_FACTOR = 1 +def read_clock(x): + timestamp = int(x, 16) + return timestamp / CLOCK_FACTOR + +def get_clock_range(data): + s = r"\[([0-9a-f]+)\] " + r = re.compile(s) + clocks = [read_clock(x) for x in r.findall(data)] + return min(clocks), max(clocks) + +def gc_collect_rows(time0, data): s = r""" ----------- Full collection ------------------ \| used before collection: @@ -84,35 +108,73 @@ r = re.compile(s.replace('\n', '')) yield 'clock', 'gc-before', 'gc-after' for a,b,c,d,e,f in r.findall(data): - yield int(f, 16), int(a)+int(b), int(c)+int(d) + clock = read_clock(f) - time0 + yield clock, int(a)+int(b), int(c)+int(d) -def tasks_rows(data): +def tasks_rows(time0, data): s = r""" -\{translation-task +\[([0-9a-f]+)\] \{translation-task starting ([\w-]+) -\[([0-9a-f]+)\] translation-task\}""" +""" # r = re.compile(s.replace('\n', '')) yield 'clock', None, 'task' for a,b in r.findall(data): - yield int(b, 16), 1, a + clock = read_clock(a) - time0 + yield clock, 1, b + -def memusage_rows(filename, maxclock): - try: - lines = open(filename) - except IOError: - print 'Warning: cannot find file %s, skipping the memusage sheet' - lines = [] - yield 'n', 'computed clock', 'VmRSS' +def loops_rows(time0, data): + s = r""" +\[([0-9a-f]+)\] \{jit-mem-looptoken-(alloc|free) +(.*?)\[ +""" + # + r = re.compile(s.replace('\n', '')) + yield 'clock', 'total', 'loops', 'bridges' + loops = 0 + bridges = 0 + fake_total = 0 + for clock, action, text in r.findall(data): + clock = read_clock(clock) - time0 + if text.startswith('allocating Loop #'): + loops += 1 + elif text.startswith('allocating Bridge #'): + bridges += 1 + elif text.startswith('freeing Loop #'): + match = re.match('freeing Loop # .* with ([0-9]*) attached bridges', text) + loops -=1 + bridges -= int(match.group(1)) + total = loops+bridges + yield clock, loops+bridges, loops, bridges + + +def vmrss_rows(filename, maxtime): + lines = [] + if options.cpython_vmrss: + try: + lines = open(filename).readlines() + except IOError: + print 'Warning: cannot find file %s, skipping this sheet' + for row in vmrss_rows_impl(lines, maxtime): + yield row + +def vmrss_rows_impl(lines, maxtime): + yield 'inferred clock', 'VmRSS' + numlines = len(lines) for i, line in enumerate(lines): mem = int(line) - yield i, "=max('gc-collect'!$A$1:$A$65536)*(A2/max($A$1:$A$65536)))", mem + clock = maxtime * i // (numlines-1) + yield clock, mem -def find_max_clock(data): - s = r"\[([0-9a-f]+)\] " - r = re.compile(s) - clocks = [int(x, 16) for x in r.findall(data)] - return max(clocks) if __name__ == '__main__': - main() + CLOCK_FACTOR = 1000000000.0 # report GigaTicks instead of Ticks + parser = optparse.OptionParser(usage="%prog logfile [options]") + parser.add_option('-c', '--cpython-vmrss', dest='cpython_vmrss', default=None, metavar='FILE', type=str, + help='the .vmrss file produced by CPython') + options, args = parser.parse_args() + if len(args) != 1: + parser.print_help() + sys.exit(2) + main(args[0], options) Modified: pypy/branch/fast-forward/pypy/module/__pypy__/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__pypy__/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/__pypy__/__init__.py Wed Dec 1 18:08:21 2010 @@ -15,6 +15,7 @@ 'debug_print' : 'interp_debug.debug_print', 'debug_stop' : 'interp_debug.debug_stop', 'debug_print_once' : 'interp_debug.debug_print_once', + 'builtinify' : 'interp_magic.builtinify', } def setup_after_space_initialization(self): Modified: pypy/branch/fast-forward/pypy/module/__pypy__/interp_magic.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__pypy__/interp_magic.py (original) +++ pypy/branch/fast-forward/pypy/module/__pypy__/interp_magic.py Wed Dec 1 18:08:21 2010 @@ -51,3 +51,9 @@ return space.newtuple([space.newint(cache.hits.get(name, 0)), space.newint(cache.misses.get(name, 0))]) mapdict_cache_counter.unwrap_spec = [ObjSpace, str] + +def builtinify(space, w_func): + from pypy.interpreter.function import Function, BuiltinFunction + func = space.interp_w(Function, w_func) + bltn = BuiltinFunction(func) + return space.wrap(bltn) Modified: pypy/branch/fast-forward/pypy/module/__pypy__/test/test_special.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__pypy__/test/test_special.py (original) +++ pypy/branch/fast-forward/pypy/module/__pypy__/test/test_special.py Wed Dec 1 18:08:21 2010 @@ -21,3 +21,18 @@ def test_cpumodel(self): import __pypy__ assert hasattr(__pypy__, 'cpumodel') + + def test_builtinify(self): + import __pypy__ + class A(object): + a = lambda *args: args + b = __pypy__.builtinify(a) + my = A() + assert my.a() == (my,) + assert my.b() == () + assert A.a(my) == (my,) + assert A.b(my) == (my,) + assert A.a.im_func(my) == (my,) + assert not hasattr(A.b, 'im_func') + assert A.a is not A.__dict__['a'] + assert A.b is A.__dict__['b'] Modified: pypy/branch/fast-forward/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/api.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/api.py Wed Dec 1 18:08:21 2010 @@ -326,6 +326,8 @@ 'PyCapsule_SetContext', 'PyCapsule_Import', 'PyCapsule_Type', 'init_capsule', 'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer', + + 'PyStructSequence_InitType', 'PyStructSequence_New', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur @@ -372,7 +374,7 @@ }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) - for cpyname in 'Method List Int Long Dict Tuple Class'.split(): + for cpyname in 'Method List Int Long Dict Class'.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } ' 'Py%sObject' % (cpyname, )) build_exported_objects() @@ -874,6 +876,7 @@ source_dir / "bufferobject.c", source_dir / "object.c", source_dir / "cobject.c", + source_dir / "structseq.c", source_dir / "capsule.c", ], separate_module_sources=separate_module_sources, Modified: pypy/branch/fast-forward/pypy/module/cpyext/cdatetime.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/cdatetime.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/cdatetime.py Wed Dec 1 18:08:21 2010 @@ -4,8 +4,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, cpython_struct, PyObjectFields) from pypy.module.cpyext.import_ import PyImport_Import -from pypy.module.cpyext.typeobject import PyTypeObjectPtr, render_immortal -from pypy.module.cpyext.state import State +from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_renamer @@ -25,48 +24,26 @@ datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw', track_allocation=False) - if not we_are_translated(): - datetimeAPI_dealloc(space) - space.fromcache(State).datetimeAPI = datetimeAPI - w_datetime = PyImport_Import(space, space.wrap("datetime")) w_type = space.getattr(w_datetime, space.wrap("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DateType, w_type) w_type = space.getattr(w_datetime, space.wrap("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DateTimeType, w_type) w_type = space.getattr(w_datetime, space.wrap("time")) datetimeAPI.c_TimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_TimeType, w_type) w_type = space.getattr(w_datetime, space.wrap("timedelta")) datetimeAPI.c_DeltaType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DeltaType, w_type) return datetimeAPI -def datetimeAPI_dealloc(space): - "Used in tests only, to please the refcount checker" - if we_are_translated(): - return - datetimeAPI = space.fromcache(State).datetimeAPI - if datetimeAPI is None: - return - space.fromcache(State).datetimeAPI = None - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DateType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DateTimeType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_TimeType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DeltaType)) - lltype.free(datetimeAPI, flavor='raw') - PyDateTime_Date = PyObject PyDateTime_Time = PyObject PyDateTime_DateTime = PyObject Modified: pypy/branch/fast-forward/pypy/module/cpyext/include/tupleobject.h ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/include/tupleobject.h (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/include/tupleobject.h Wed Dec 1 18:08:21 2010 @@ -10,9 +10,19 @@ /* defined in varargswrapper.c */ PyObject * PyTuple_Pack(Py_ssize_t, ...); -#define PyTuple_SET_ITEM PyTuple_SetItem -#define PyTuple_GET_ITEM PyTuple_GetItem +typedef struct { + PyObject_HEAD + PyObject **items; + Py_ssize_t size; +} PyTupleObject; +#define PyTuple_GET_ITEM PyTuple_GetItem + +/* Macro, trading safety for speed */ +#define PyTuple_GET_SIZE(op) (((PyTupleObject *)(op))->size) + +/* Macro, *only* to be used to fill in brand new tuples */ +#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->items[i] = v) #ifdef __cplusplus } Modified: pypy/branch/fast-forward/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/sequence.py Wed Dec 1 18:08:21 2010 @@ -116,6 +116,14 @@ This is the equivalent of the Python expression o1 + o2.""" return space.add(w_o1, w_o2) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PySequence_Contains(space, w_obj, w_value): + """Determine if o contains value. If an item in o is equal to value, + return 1, otherwise return 0. On error, return -1. This is + equivalent to the Python expression value in o.""" + w_res = space.contains(w_obj, w_value) + return space.int_w(w_res) + @cpython_api([PyObject], PyObject) def PySeqIter_New(space, w_seq): """Return an iterator that works with a general sequence object, seq. The Modified: pypy/branch/fast-forward/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/slotdefs.py Wed Dec 1 18:08:21 2010 @@ -6,7 +6,7 @@ unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, setattrofunc, lenfunc, ssizeargfunc, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, hashfunc, - descrgetfunc, descrsetfunc) + descrgetfunc, descrsetfunc, objobjproc) from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -28,14 +28,14 @@ def check_num_args(space, ob, n): from pypy.module.cpyext.tupleobject import PyTuple_CheckExact, \ - PyTuple_GET_SIZE + _PyTuple_Size_Fast if not PyTuple_CheckExact(space, ob): raise OperationError(space.w_SystemError, space.wrap("PyArg_UnpackTuple() argument list is not a tuple")) - if n == PyTuple_GET_SIZE(space, ob): + if n == _PyTuple_Size_Fast(space, ob): return raise operationerrfmt(space.w_TypeError, - "expected %d arguments, got %d", n, PyTuple_GET_SIZE(space, ob)) + "expected %d arguments, got %d", n, _PyTuple_Size_Fast(space, ob)) def wrap_init(space, w_self, w_args, func, w_kwargs): func_init = rffi.cast(initproc, func) @@ -156,6 +156,15 @@ if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) +def wrap_objobjproc(space, w_self, w_args, func): + func_target = rffi.cast(objobjproc, func) + check_num_args(space, w_args, 1) + w_value, = space.fixedview(w_args) + res = generic_cpy_call(space, func_target, w_self, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.wrap(res) + def wrap_ssizessizeargfunc(space, w_self, w_args, func): func_target = rffi.cast(ssizessizeargfunc, func) check_num_args(space, w_args, 2) Modified: pypy/branch/fast-forward/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/state.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/state.py Wed Dec 1 18:08:21 2010 @@ -6,8 +6,6 @@ import sys class State: - datetimeAPI = None # used in tests - def __init__(self, space): self.space = space self.reset() Modified: pypy/branch/fast-forward/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/stubs.py Wed Dec 1 18:08:21 2010 @@ -2254,13 +2254,6 @@ in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PySequence_Contains(space, o, value): - """Determine if o contains value. If an item in o is equal to value, - return 1, otherwise return 0. On error, return -1. This is - equivalent to the Python expression value in o.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PySequence_Index(space, o, value): """Return the first index i for which o[i] == value. On error, return Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_arraymodule.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_arraymodule.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_arraymodule.py Wed Dec 1 18:08:21 2010 @@ -14,6 +14,7 @@ arr.append(4) assert arr.tolist() == [1, 2, 3, 4] assert len(arr) == 4 + self.cleanup_references() def test_iter(self): module = self.import_module(name='array') @@ -22,6 +23,7 @@ for i in arr: sum += i assert sum == 6 + self.cleanup_references() def test_index(self): module = self.import_module(name='array') @@ -32,6 +34,7 @@ assert arr.tolist() == [1,2,4] arr[2] = 99 assert arr.tolist() == [1,2,99] + self.cleanup_references() def test_slice_get(self): module = self.import_module(name='array') @@ -40,3 +43,4 @@ assert arr[1:].tolist() == [2,3,4] assert arr[:2].tolist() == [1,2] assert arr[1:3].tolist() == [2,3] + self.cleanup_references() Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py Wed Dec 1 18:08:21 2010 @@ -1,10 +1,12 @@ import sys +import weakref import os.path import py from pypy.conftest import gettestobjspace from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import interp2app from pypy.rpython.lltypesystem import rffi, lltype, ll2ctypes from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator import platform @@ -88,6 +90,23 @@ self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) class LeakCheckingTest(object): + @staticmethod + def cleanup_references(space): + state = space.fromcache(RefcountState) + + import gc; gc.collect() + # Clear all lifelines, objects won't resurrect + for w_obj, obj in state.lifeline_dict._dict.items(): + if w_obj not in state.py_objects_w2r: + state.lifeline_dict.set(w_obj, None) + del obj + import gc; gc.collect() + + for w_obj in state.non_heaptypes_w: + Py_DecRef(space, w_obj) + state.non_heaptypes_w[:] = [] + state.reset_borrowed_references() + def check_and_print_leaks(self): # check for sane refcnts import gc @@ -98,13 +117,6 @@ lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) - # Clear all lifelines, objects won't resurrect - for w_obj, obj in state.lifeline_dict._dict.items(): - if w_obj not in state.py_objects_w2r: - state.lifeline_dict.set(w_obj, None) - del obj - gc.collect() - for w_obj, obj in state.py_objects_w2r.iteritems(): base_refcnt = self.frozen_refcounts.get(w_obj) delta = obj.c_ob_refcnt @@ -114,7 +126,12 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = state.lifeline_dict.get(w_obj) + try: + weakref.ref(w_obj) + except TypeError: + lifeline = None + else: + lifeline = state.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: @@ -146,6 +163,8 @@ state = cls.space.fromcache(RefcountState) state.non_heaptypes_w[:] = [] + cls.w_cleanup_references = cls.space.wrap(interp2app(cls.cleanup_references)) + def compile_module(self, name, **kwds): """ Build an extension module linked against the cpyext api library. @@ -265,13 +284,7 @@ def teardown_method(self, func): for name in self.imported_module_names: self.unimport_module(name) - state = self.space.fromcache(RefcountState) - for w_obj in state.non_heaptypes_w: - Py_DecRef(self.space, w_obj) - state.non_heaptypes_w[:] = [] - state.reset_borrowed_references() - from pypy.module.cpyext import cdatetime - cdatetime.datetimeAPI_dealloc(self.space) + self.cleanup_references(self.space) if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." @@ -541,16 +554,17 @@ PyObject *true = Py_True; PyObject *tup = NULL; int refcnt = true->ob_refcnt; - int refcnt_after; + int refcnt_middle, refcnt_after; tup = PyTuple_New(1); Py_INCREF(true); if (PyTuple_SetItem(tup, 0, true) < 0) return NULL; - refcnt_after = true->ob_refcnt; + refcnt_middle = true->ob_refcnt; Py_DECREF(tup); - fprintf(stderr, "REFCNT2 %i %i\\n", refcnt, refcnt_after); - return PyBool_FromLong(refcnt_after == refcnt); + refcnt_after = true->ob_refcnt; + fprintf(stderr, "REFCNT2 %i %i %i\\n", refcnt, refcnt_middle, refcnt_after); + return PyBool_FromLong(refcnt_after == refcnt && refcnt_middle == refcnt+1); } static PyMethodDef methods[] = { Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_datetime.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_datetime.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_datetime.py Wed Dec 1 18:08:21 2010 @@ -92,9 +92,20 @@ PyDateTimeAPI->TimeType, PyDateTimeAPI->DeltaType); """), + ("clear_types", "METH_NOARGS", + """ + Py_DECREF(PyDateTimeAPI->DateType); + Py_DECREF(PyDateTimeAPI->DateTimeType); + Py_DECREF(PyDateTimeAPI->TimeType); + Py_DECREF(PyDateTimeAPI->DeltaType); + Py_RETURN_NONE; + """ + ) ]) import datetime assert module.get_types() == (datetime.date, datetime.datetime, datetime.time, datetime.timedelta) + module.clear_types() + self.cleanup_references() Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_sequence.py Wed Dec 1 18:08:21 2010 @@ -70,3 +70,11 @@ assert space.unwrap(space.next(w_iter)) == 2 exc = raises(OperationError, space.next, w_iter) assert exc.value.match(space, space.w_StopIteration) + + def test_contains(self, space, api): + w_t = space.wrap((1, 'ha')) + assert api.PySequence_Contains(w_t, space.wrap(u'ha')) + assert not api.PySequence_Contains(w_t, space.wrap(2)) + assert api.PySequence_Contains(space.w_None, space.wrap(2)) == -1 + assert api.PyErr_Occurred() + api.PyErr_Clear() Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_tupleobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_tupleobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_tupleobject.py Wed Dec 1 18:08:21 2010 @@ -7,24 +7,34 @@ class TestTupleObject(BaseApiTest): def test_tupleobject(self, space, api): assert not api.PyTuple_Check(space.w_None) - assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 + #assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 XXX atuple = space.newtuple([0, 1, 'yay']) assert api.PyTuple_Size(atuple) == 3 - assert api.PyTuple_GET_SIZE(atuple) == 3 - raises(TypeError, api.PyTuple_Size(space.newlist([]))) + #raises(TypeError, api.PyTuple_Size(space.newlist([]))) XXX api.PyErr_Clear() def test_tuple_resize(self, space, api): - py_tuple = api.PyTuple_New(3) + ref_tup = api.PyTuple_New(3) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - ar[0] = rffi.cast(PyObject, make_ref(space, py_tuple)) + ar[0] = rffi.cast(PyObject, ref_tup) api._PyTuple_Resize(ar, 2) - py_tuple = from_ref(space, ar[0]) - assert len(py_tuple.wrappeditems) == 2 + assert ar[0] == rffi.cast(PyObject, ref_tup) + # ^^^ our _PyTuple_Resize does not actually need to change the ptr so far + assert api.PyTuple_Size(ar[0]) == 2 api._PyTuple_Resize(ar, 10) - py_tuple = from_ref(space, ar[0]) - assert len(py_tuple.wrappeditems) == 10 + assert api.PyTuple_Size(ar[0]) == 10 api.Py_DecRef(ar[0]) lltype.free(ar, flavor='raw') + + def test_tuple_setup(self, space, api): + ref_tup = api.PyTuple_New(2) + ref0 = make_ref(space, space.wrap(123)) + api.PyTuple_SetItem(ref_tup, 0, ref0) + ref1 = make_ref(space, space.wrap(456)) + api.PyTuple_SetItem(ref_tup, 1, ref1) + + w_tup = from_ref(space, ref_tup) + assert space.is_true(space.eq(w_tup, space.wrap((123, 456)))) + api.Py_DecRef(ref_tup) Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_typeobject.py Wed Dec 1 18:08:21 2010 @@ -20,6 +20,7 @@ assert type(obj) is module.fooType print "type of obj has type", type(type(obj)) print "type of type of obj has type", type(type(type(obj))) + self.cleanup_references() def test_typeobject_method_descriptor(self): module = self.import_module(name='foo') @@ -38,6 +39,7 @@ print obj.foo assert obj.foo == 42 assert obj.int_member == obj.foo + self.cleanup_references() def test_typeobject_data_member(self): module = self.import_module(name='foo') @@ -54,6 +56,7 @@ raises(SystemError, "obj.broken_member = 42") assert module.fooType.broken_member.__doc__ is None assert module.fooType.object_member.__doc__ == "A Python object." + self.cleanup_references() def test_typeobject_object_member(self): module = self.import_module(name='foo') @@ -74,6 +77,7 @@ obj.set_foo = 32 assert obj.foo == 32 + self.cleanup_references() def test_typeobject_string_member(self): module = self.import_module(name='foo') @@ -91,6 +95,7 @@ assert obj.char_member == "a" raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") + self.cleanup_references() def test_staticmethod(self): module = self.import_module(name="foo") @@ -98,6 +103,7 @@ assert obj.foo == 42 obj2 = obj.create() assert obj2.foo == 42 + self.cleanup_references() def test_new(self): module = self.import_module(name='foo') @@ -118,7 +124,8 @@ return self assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) - + self.cleanup_references() + def test_init(self): module = self.import_module(name="foo") newobj = module.FuuType() @@ -137,6 +144,7 @@ newobj = Fuu2() assert newobj.get_val() == 42 assert newobj.foobar == 32 + self.cleanup_references() def test_metatype(self): module = self.import_module(name='foo') @@ -145,6 +153,7 @@ assert isinstance(x, type) assert isinstance(x, module.MetaType) x() + self.cleanup_references() def test_metaclass_compatible(self): # metaclasses should not conflict here @@ -153,7 +162,9 @@ assert type(module.fooType).__mro__ == (type, object) y = module.MetaType('other', (module.fooType,), {}) assert isinstance(y, module.MetaType) - y() + x = y() + del x, y + self.cleanup_references() def test_sre(self): module = self.import_module(name='_sre') @@ -172,17 +183,21 @@ assert "groupdict" in dir(m) re._cache.clear() re._cache_repl.clear() + del prog, m + self.cleanup_references() def test_init_error(self): module = self.import_module("foo") raises(ValueError, module.InitErrType) - + self.cleanup_references() + def test_cmps(self): module = self.import_module("comparisons") cmpr = module.CmpType() assert cmpr == 3 assert cmpr != 42 - + self.cleanup_references() + def test_hash(self): module = self.import_module("comparisons") cmpr = module.CmpType() @@ -191,6 +206,7 @@ d[cmpr] = 72 assert d[cmpr] == 72 assert d[3] == 72 + self.cleanup_references() def test_descriptor(self): module = self.import_module("foo") @@ -205,6 +221,7 @@ assert obj.y == (prop, 2) del obj.x assert obj.z == prop + self.cleanup_references() def test_tp_dict(self): foo = self.import_module("foo") @@ -226,6 +243,7 @@ ]) obj = foo.new() assert module.read_tp_dict(obj) == foo.fooType.copy + self.cleanup_references() class TestTypes(BaseApiTest): Modified: pypy/branch/fast-forward/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/tupleobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/tupleobject.py Wed Dec 1 18:08:21 2010 @@ -1,55 +1,144 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers) + build_type_checkers, PyObjectFields, + cpython_struct, bootstrap_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - borrow_from, make_ref, from_ref) + borrow_from, make_ref, from_ref, make_typedescr, get_typedescr, Reference, + track_reference) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall -from pypy.objspace.std.tupleobject import W_TupleObject +## +## Implementation of PyTupleObject +## =============================== +## +## We have the same problem as PyStringObject: a PyTupleObject can be +## initially used in a read-write way with PyTuple_New(), PyTuple_SetItem() +## and _PyTuple_Resize(). +## +## The 'size' and 'items' fields of a PyTupleObject are always valid. +## Apart from that detail, see the big comment in stringobject.py for +## more information. +## + +ARRAY_OF_PYOBJ = rffi.CArrayPtr(PyObject) +PyTupleObjectStruct = lltype.ForwardReference() +PyTupleObject = lltype.Ptr(PyTupleObjectStruct) +PyTupleObjectFields = PyObjectFields + \ + (("items", ARRAY_OF_PYOBJ), ("size", Py_ssize_t)) +cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) + + at bootstrap_function +def init_tupleobject(space): + "Type description of PyTupleObject" + make_typedescr(space.w_tuple.instancetypedef, + basestruct=PyTupleObject.TO, + attach=tuple_attach, + dealloc=tuple_dealloc, + realize=tuple_realize) PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple") +def new_empty_tuple(space, length): + """ + Allocate a PyTupleObject and its array, but without a corresponding + interpreter object. The array items may be mutated, until + tuple_realize() is called. + """ + typedescr = get_typedescr(space.w_tuple.instancetypedef) + py_obj = typedescr.allocate(space, space.w_tuple) + py_tup = rffi.cast(PyTupleObject, py_obj) + + py_tup.c_items = lltype.malloc(ARRAY_OF_PYOBJ.TO, length, + flavor='raw', zero=True) + py_tup.c_size = length + return py_tup + +def tuple_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PyTupleObject with the given tuple object. + """ + items_w = space.fixedview(w_obj) + py_tup = rffi.cast(PyTupleObject, py_obj) + py_tup.c_items = lltype.nullptr(ARRAY_OF_PYOBJ.TO) + py_tup.c_size = len(items_w) + +def tuple_realize(space, py_obj): + """ + Creates the tuple in the interpreter. The PyTupleObject items array + must not be modified after this call. + """ + py_tup = rffi.cast(PyTupleObject, py_obj) + # If your CPython extension creates a self-referential tuple + # with PyTuple_SetItem(), you loose. + c_items = py_tup.c_items + items_w = [from_ref(space, c_items[i]) for i in range(py_tup.c_size)] + w_obj = space.newtuple(items_w) + track_reference(space, py_obj, w_obj) + return w_obj + + at cpython_api([PyObject], lltype.Void, external=False) +def tuple_dealloc(space, py_obj): + """Frees allocated PyTupleObject resources. + """ + py_tup = rffi.cast(PyTupleObject, py_obj) + if py_tup.c_items: + for i in range(py_tup.c_size): + Py_DecRef(space, py_tup.c_items[i]) + lltype.free(py_tup.c_items, flavor="raw") + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) + +#_______________________________________________________________________ + @cpython_api([Py_ssize_t], PyObject) def PyTuple_New(space, size): - return space.newtuple([space.w_None] * size) + return rffi.cast(PyObject, new_empty_tuple(space, size)) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PyTuple_SetItem(space, w_t, pos, w_obj): - if not PyTuple_Check(space, w_t): - # XXX this should also steal a reference, test it!!! - PyErr_BadInternalCall(space) - assert isinstance(w_t, W_TupleObject) - w_t.wrappeditems[pos] = w_obj - Py_DecRef(space, w_obj) # SetItem steals a reference! +def PyTuple_SetItem(space, ref, pos, ref_item): + # XXX steals a reference at the level of PyObjects. Don't try to + # XXX call this function with an interpreter object as ref_item! + + # XXX do PyTuple_Check, without forcing ref as an interpreter object + # XXX -- then if it fails it should also steal a reference, test it!!! + ref_tup = rffi.cast(PyTupleObject, ref) + if not ref_tup.c_items: + msg = "PyTuple_SetItem() called on an already-escaped tuple object" + raise OperationError(space.w_SystemError, space.wrap(msg)) + ref_old = ref_tup.c_items[pos] + ref_tup.c_items[pos] = ref_item # SetItem steals a reference! + Py_DecRef(space, ref_old) return 0 @cpython_api([PyObject, Py_ssize_t], PyObject) -def PyTuple_GetItem(space, w_t, pos): - if not PyTuple_Check(space, w_t): - PyErr_BadInternalCall(space) - assert isinstance(w_t, W_TupleObject) - w_obj = w_t.wrappeditems[pos] - return borrow_from(w_t, w_obj) - - at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) -def PyTuple_GET_SIZE(space, w_t): - """Return the size of the tuple p, which must be non-NULL and point to a tuple; - no error checking is performed. """ - assert isinstance(w_t, W_TupleObject) - return len(w_t.wrappeditems) +def PyTuple_GetItem(space, ref, pos): + # XXX do PyTuple_Check, without forcing ref as an interpreter object + ref_tup = rffi.cast(PyTupleObject, ref) + if ref_tup.c_items: + return Reference(ref_tup.c_items[pos]) # borrowed reference + else: + w_t = from_ref(space, ref) + w_obj = space.getitem(w_t, space.wrap(pos)) + return borrow_from(w_t, w_obj) + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def _PyTuple_Size_Fast(space, ref): + # custom version: it's not a macro, so it can be called from other .py + # files; but it doesn't include PyTuple_Check() + ref_tup = rffi.cast(PyTupleObject, ref) + return ref_tup.c_size @cpython_api([PyObject], Py_ssize_t, error=-1) def PyTuple_Size(space, ref): """Take a pointer to a tuple object, and return the size of that tuple.""" - if not PyTuple_Check(space, ref): - raise OperationError(space.w_TypeError, - space.wrap("expected tuple object")) - return PyTuple_GET_SIZE(space, ref) + # XXX do PyTuple_Check, without forcing ref as an interpreter object + ref_tup = rffi.cast(PyTupleObject, ref) + return ref_tup.c_size @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyTuple_Resize(space, ref, newsize): +def _PyTuple_Resize(space, refp, newsize): """Can be used to resize a tuple. newsize will be the new length of the tuple. Because tuples are supposed to be immutable, this should only be used if there is only one reference to the object. Do not use this if the tuple may already @@ -60,18 +149,22 @@ this function. If the object referenced by *p is replaced, the original *p is destroyed. On failure, returns -1 and sets *p to NULL, and raises MemoryError or SystemError.""" - py_tuple = from_ref(space, ref[0]) - if not PyTuple_Check(space, py_tuple): - PyErr_BadInternalCall(space) - assert isinstance(py_tuple, W_TupleObject) - py_newtuple = PyTuple_New(space, newsize) - - to_cp = newsize - oldsize = len(py_tuple.wrappeditems) - if oldsize < newsize: - to_cp = oldsize - for i in range(to_cp): - py_newtuple.wrappeditems[i] = py_tuple.wrappeditems[i] - Py_DecRef(space, ref[0]) - ref[0] = make_ref(space, py_newtuple) + # XXX do PyTuple_Check, without forcing ref as an interpreter object + # XXX -- then if it fails it should reset refp[0] to null + ref_tup = rffi.cast(PyTupleObject, refp[0]) + c_newitems = lltype.malloc(ARRAY_OF_PYOBJ.TO, newsize, + flavor='raw', zero=True) + c_olditems = ref_tup.c_items + if not c_olditems: + msg = "_PyTuple_Resize() called on an already-escaped tuple object" + raise OperationError(space.w_SystemError, space.wrap(msg)) + oldsize = ref_tup.c_size + for i in range(min(oldsize, newsize)): + c_newitems[i] = c_olditems[i] + # decref items deleted by shrinkage + for i in range(newsize, oldsize): + Py_DecRef(space, c_olditems[i]) + ref_tup.c_items = c_newitems + ref_tup.c_size = newsize + lltype.free(c_olditems, flavor='raw') return 0 Modified: pypy/branch/fast-forward/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/typeobject.py Wed Dec 1 18:08:21 2010 @@ -24,7 +24,7 @@ from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, - PyNumberMethods) + PyNumberMethods, PySequenceMethods) from pypy.module.cpyext.slotdefs import ( slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) from pypy.interpreter.error import OperationError @@ -136,6 +136,8 @@ if not struct: if slot_names[0] == 'c_tp_as_number': STRUCT_TYPE = PyNumberMethods + elif slot_names[0] == 'c_tp_as_sequence': + STRUCT_TYPE = PySequenceMethods else: raise AssertionError( "Structure not allocated: %s" % (slot_names[0],)) @@ -392,6 +394,8 @@ lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') if obj_pto.c_tp_as_number: lltype.free(obj_pto.c_tp_as_number, flavor='raw') + if obj_pto.c_tp_as_sequence: + lltype.free(obj_pto.c_tp_as_sequence, flavor='raw') Py_DecRef(space, base_pyo) rffi.free_charp(obj_pto.c_tp_name) PyObject_dealloc(space, obj) @@ -433,12 +437,6 @@ finish_type_1(space, pto) finish_type_2(space, pto, w_type) - if space.type(w_type).is_cpytype(): - # XXX Types with a C metatype are never freed, try to see why... - render_immortal(pto, w_type) - lltype.render_immortal(pto) - lltype.render_immortal(pto.c_tp_name) - pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: @@ -544,25 +542,12 @@ w_obj.ready() finish_type_2(space, py_type, w_obj) - render_immortal(py_type, w_obj) state = space.fromcache(RefcountState) state.non_heaptypes_w.append(w_obj) return w_obj -def render_immortal(py_type, w_obj): - lltype.render_immortal(py_type.c_tp_bases) - lltype.render_immortal(py_type.c_tp_mro) - - assert isinstance(w_obj, W_TypeObject) - if w_obj.is_cpytype(): - lltype.render_immortal(py_type.c_tp_dict) - else: - lltype.render_immortal(py_type.c_tp_name) - if not w_obj.is_cpytype() and w_obj.is_heaptype(): - lltype.render_immortal(py_type) - def finish_type_1(space, pto): """ Sets up tp_bases, necessary before creating the interpreter type. @@ -599,7 +584,8 @@ if w_obj.is_cpytype(): Py_DecRef(space, pto.c_tp_dict) - pto.c_tp_dict = make_ref(space, w_obj.getdict()) + w_dict = space.newdict(from_strdict_shared=w_obj.dict_w) + pto.c_tp_dict = make_ref(space, w_dict) @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): Modified: pypy/branch/fast-forward/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/branch/fast-forward/pypy/module/fcntl/test/test_fcntl.py Wed Dec 1 18:08:21 2010 @@ -63,7 +63,7 @@ if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', 'Darwin1.2', 'darwin', 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', + 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', 'bsdos2', 'bsdos3', 'bsdos4', 'openbsd', 'openbsd2', 'openbsd3'): if struct.calcsize('l') == 8: @@ -159,7 +159,7 @@ if "linux" in sys.platform: TIOCGPGRP = 0x540f - elif "darwin" in sys.platform or "freebsd6" == sys.platform: + elif "darwin" in sys.platform or "freebsd" in sys.platform: TIOCGPGRP = 0x40047477 else: skip("don't know how to test ioctl() on this platform") Modified: pypy/branch/fast-forward/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/imp/importing.py (original) +++ pypy/branch/fast-forward/pypy/module/imp/importing.py Wed Dec 1 18:08:21 2010 @@ -77,7 +77,7 @@ return SEARCH_ERROR, None, None -if sys.platform in ['linux2', 'freebsd']: +if sys.platform == 'linux2' or 'freebsd' in sys.platform: def case_ok(filename): return True else: Modified: pypy/branch/fast-forward/pypy/module/itertools/interp_itertools.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/itertools/interp_itertools.py (original) +++ pypy/branch/fast-forward/pypy/module/itertools/interp_itertools.py Wed Dec 1 18:08:21 2010 @@ -394,18 +394,20 @@ raise OperationError(self.space.w_StopIteration, self.space.w_None) if not self.w_it: self._advance() + try: + return self.space.next(self.w_it) + except OperationError, e: + return self._handle_error(e) + def _handle_error(self, e): while True: + if not e.match(self.space, self.space.w_StopIteration): + raise e + self._advance() # may raise StopIteration itself try: - w_obj = self.space.next(self.w_it) + return self.space.next(self.w_it) except OperationError, e: - if e.match(self.space, self.space.w_StopIteration): - self._advance() # may raise StopIteration itself - else: - raise - else: - break - return w_obj + pass # loop back to the start of _handle_error(e) def W_Chain___new__(space, w_subtype, args_w): w_args = space.newtuple(args_w) @@ -443,8 +445,10 @@ def __init__(self, space, w_fun, args_w): self.space = space - self.identity_fun = (self.space.is_w(w_fun, space.w_None)) - self.w_fun = w_fun + if self.space.is_w(w_fun, space.w_None): + self.w_fun = None + else: + self.w_fun = w_fun iterators_w = [] i = 0 @@ -467,12 +471,26 @@ return self.space.wrap(self) def next_w(self): - w_objects = self.space.newtuple([self.space.next(w_it) for w_it in self.iterators_w]) - if self.identity_fun: + # common case: 1 or 2 arguments + iterators_w = self.iterators_w + length = len(iterators_w) + if length == 1: + objects = [self.space.next(iterators_w[0])] + elif length == 2: + objects = [self.space.next(iterators_w[0]), + self.space.next(iterators_w[1])] + else: + objects = self._get_objects() + w_objects = self.space.newtuple(objects) + if self.w_fun is None: return w_objects else: return self.space.call(self.w_fun, w_objects) + def _get_objects(self): + # the loop is out of the way of the JIT + return [self.space.next(w_elem) for w_elem in self.iterators_w] + def W_IMap___new__(space, w_subtype, w_fun, args_w): if len(args_w) == 0: @@ -827,15 +845,7 @@ raise OperationError(self.space.w_StopIteration, self.space.w_None) if not self.new_group: - # Consume unwanted input until we reach the next group - try: - while True: - self.group_next(self.index) - - except StopIteration: - pass - if self.exhausted: - raise OperationError(self.space.w_StopIteration, self.space.w_None) + self._consume_unwanted_input() if not self.started: self.started = True @@ -857,6 +867,16 @@ w_iterator = self.space.wrap(W_GroupByIterator(self.space, self.index, self)) return self.space.newtuple([self.w_key, w_iterator]) + def _consume_unwanted_input(self): + # Consume unwanted input until we reach the next group + try: + while True: + self.group_next(self.index) + except StopIteration: + pass + if self.exhausted: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + def group_next(self, group_index): if group_index < self.index: raise StopIteration Modified: pypy/branch/fast-forward/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/__init__.py Wed Dec 1 18:08:21 2010 @@ -96,6 +96,8 @@ interpleveldefs['fork'] = 'interp_posix.fork' if hasattr(os, 'openpty'): interpleveldefs['openpty'] = 'interp_posix.openpty' + if hasattr(os, 'forkpty'): + interpleveldefs['forkpty'] = 'interp_posix.forkpty' if hasattr(os, 'waitpid'): interpleveldefs['waitpid'] = 'interp_posix.waitpid' if hasattr(os, 'execv'): Modified: pypy/branch/fast-forward/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/interp_posix.py Wed Dec 1 18:08:21 2010 @@ -653,6 +653,14 @@ raise wrap_oserror(space, e) return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)]) +def forkpty(space): + try: + pid, master_fd = os.forkpty() + except OSError, e: + raise wrap_oserror(space, e) + return space.newtuple([space.wrap(pid), + space.wrap(master_fd)]) + def waitpid(space, pid, options): """ waitpid(pid, options) -> (pid, status) Modified: pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py Wed Dec 1 18:08:21 2010 @@ -333,6 +333,22 @@ data = os.read(master_fd, 100) assert data.startswith('x') + if hasattr(__import__(os.name), "forkpty"): + def test_forkpty(self): + import sys + os = self.posix + childpid, master_fd = os.forkpty() + assert isinstance(childpid, int) + assert isinstance(master_fd, int) + if childpid == 0: + data = os.read(0, 100) + if data.startswith('abc'): + os._exit(42) + else: + os._exit(43) + os.write(master_fd, 'abc\n') + _, status = os.waitpid(childpid, 0) + assert status >> 8 == 42 if hasattr(__import__(os.name), "execv"): def test_execv(self): Modified: pypy/branch/fast-forward/pypy/module/pyexpat/test/test_parser.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/pyexpat/test/test_parser.py (original) +++ pypy/branch/fast-forward/pypy/module/pyexpat/test/test_parser.py Wed Dec 1 18:08:21 2010 @@ -16,6 +16,20 @@ raises(pyexpat.ExpatError, p.Parse, "3") + def test_encoding(self): + import pyexpat + for encoding_arg in (None, 'utf-8', 'iso-8859-1'): + for namespace_arg in (None, '{'): + print encoding_arg, namespace_arg + p = pyexpat.ParserCreate(encoding_arg, namespace_arg) + data = [] + p.CharacterDataHandler = lambda s: data.append(s) + encoding = encoding_arg is None and 'utf-8' or encoding_arg + + res = p.Parse(u"\u00f6".encode(encoding), isfinal=True) + assert res == 1 + assert data == [u"\u00f6"] + def test_intern(self): import pyexpat p = pyexpat.ParserCreate() Modified: pypy/branch/fast-forward/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/fast-forward/pypy/module/pypyjit/policy.py Wed Dec 1 18:08:21 2010 @@ -12,19 +12,13 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys', 'array', '_ffi']: + 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator']: return True return False def look_inside_function(self, func): - # this function should never actually return True directly - # but instead call the base implementation mod = func.__module__ or '?' - if mod.startswith('pypy.objspace.'): - # gc_id operation - if func.__name__ == 'id__ANY': - return False if mod == 'pypy.rlib.rbigint' or mod == 'pypy.rlib.rlocale': return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff Modified: pypy/branch/fast-forward/pypy/module/pypyjit/test/test_policy.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/pypyjit/test/test_policy.py (original) +++ pypy/branch/fast-forward/pypy/module/pypyjit/test/test_policy.py Wed Dec 1 18:08:21 2010 @@ -4,7 +4,7 @@ def test_id_any(): from pypy.objspace.std.default import id__ANY - assert not pypypolicy.look_inside_function(id__ANY) + assert pypypolicy.look_inside_function(id__ANY) def test_bigint(): from pypy.rlib.rbigint import rbigint Modified: pypy/branch/fast-forward/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/fast-forward/pypy/module/pypyjit/test/test_pypy_c.py Wed Dec 1 18:08:21 2010 @@ -630,6 +630,32 @@ ''', 3000, ([0], 2000*3)) assert len(self.loops) == 1 + def test_getattr_with_dynamic_attribute(self): + self.run_source(''' + class A(object): + pass + + l = ["x", "y"] + + def main(arg): + sum = 0 + a = A() + a.a1 = 0 + a.a2 = 0 + a.a3 = 0 + a.a4 = 0 + a.a5 = 0 # workaround, because the first five attributes need a promotion + a.x = 1 + a.y = 2 + i = 0 + while i < 2000: + name = l[i % 2] + sum += getattr(a, name) + i += 1 + return sum + ''', 3000, ([0], 3000)) + assert len(self.loops) == 1 + def test_blockstack_virtualizable(self): self.run_source(''' from pypyjit import residual_call @@ -673,11 +699,9 @@ i = t2[3] del t2 return i - ''', 100, ([], 100)) + ''', 40, ([], 100)) bytecode, = self.get_by_bytecode('BINARY_SUBSCR') - assert len(bytecode.get_opnames('new_array')) == 1 - # XXX I would like here to say that it's 0, but unfortunately - # call that can raise is not exchanged into getarrayitem_gc + assert len(bytecode.get_opnames('new_array')) == 0 def test_overflow_checking(self): startvalue = sys.maxint - 2147483647 Modified: pypy/branch/fast-forward/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/__init__.py Wed Dec 1 18:08:21 2010 @@ -65,8 +65,6 @@ 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', 'hexversion' : 'version.get_hexversion(space)', - 'ps1' : 'space.wrap(">>>> ")', - 'ps2' : 'space.wrap(".... ")', 'displayhook' : 'hook.displayhook', '__displayhook__' : 'hook.__displayhook__', Modified: pypy/branch/fast-forward/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/mapdict.py Wed Dec 1 18:08:21 2010 @@ -42,11 +42,23 @@ return None def index(self, selector): - if (self.space.config.objspace.std.withmethodcache and - not jit.we_are_jitted()): - return self._index_cache(selector) + if jit.we_are_jitted(): + # hack for the jit: + # the _index method is pure too, but its argument is never + # constant, because it is always a new tuple + return self._index_jit_pure(selector[0], selector[1]) else: - return self._index(selector) + return self._index_indirection(selector) + + @jit.purefunction + def _index_jit_pure(self, name, index): + return self._index_indirection((name, index)) + + @jit.dont_look_inside + def _index_indirection(self, selector): + if (self.space.config.objspace.std.withmethodcache): + return self._index_cache(selector) + return self._index(selector) @jit.dont_look_inside def _index_cache(self, selector): @@ -498,10 +510,11 @@ def _mapdict_read_storage(self, index): assert index >= 0 - for i in rangenmin1: - if index == i: - erased = getattr(self, "_value%s" % i) - return rerased.unerase(erased, W_Root) + if index < nmin1: + for i in rangenmin1: + if index == i: + erased = getattr(self, "_value%s" % i) + return rerased.unerase(erased, W_Root) if self._has_storage_list(): return self._mapdict_get_storage_list()[index - nmin1] erased = getattr(self, "_value%s" % nmin1) Modified: pypy/branch/fast-forward/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/objspace.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/objspace.py Wed Dec 1 18:08:21 2010 @@ -9,7 +9,7 @@ from pypy.objspace.descroperation import DescrOperation, raiseattrerror from pypy.rlib.objectmodel import instantiate, r_dict, specialize from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib.rarithmetic import base_int +from pypy.rlib.rarithmetic import base_int, widen from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.jit import hint from pypy.tool.sourcetools import func_with_new_name @@ -176,7 +176,11 @@ #print 'wrapping', x, '->', w_result return w_result if isinstance(x, base_int): - return W_LongObject.fromrarith_int(x) + x = widen(x) + if isinstance(x, int): + return self.newint(x) + else: + return W_LongObject.fromrarith_int(x) # _____ below here is where the annotator should not get _____ @@ -372,7 +376,7 @@ self, w_obj, expected_length)[:]) if expected_length != -1 and len(t) != expected_length: raise self._wrap_expected_length(expected_length, len(t)) - return t + return make_sure_not_resized(t) def fixedview_unroll(self, w_obj, expected_length=-1): return self.fixedview(w_obj, expected_length, unroll=True) Modified: pypy/branch/fast-forward/pypy/rlib/debug.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/debug.py (original) +++ pypy/branch/fast-forward/pypy/rlib/debug.py Wed Dec 1 18:08:21 2010 @@ -226,33 +226,6 @@ hop.exception_cannot_occur() return hop.inputarg(hop.args_r[0], arg=0) -def list_not_modified_any_more(arg): - """ Returns an annotator-time copy of the list 'arg' which is - flagged as 'don't mutate me'. Actually implemented as just - returning 'arg'. This is useful for debugging only. - """ - return arg - -class Entry(ExtRegistryEntry): - _about_ = list_not_modified_any_more - - def compute_result_annotation(self, s_arg): - from pypy.annotation.model import SomeList - assert isinstance(s_arg, SomeList) - # the logic behind it is that we try not to propagate - # make_sure_not_resized, when list comprehension is not on - if self.bookkeeper.annotator.translator.config.translation.list_comprehension_operations: - s_arg = s_arg.listdef.offspring() - s_arg.listdef.never_mutate() - else: - from pypy.annotation.annrpython import log - log.WARNING('list_not_modified_any_more called, but has no effect since list_comprehension is off') - return s_arg - - def specialize_call(self, hop): - hop.exception_cannot_occur() - return hop.inputarg(hop.args_r[0], arg=0) - class IntegerCanBeNegative(Exception): pass Modified: pypy/branch/fast-forward/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/libffi.py (original) +++ pypy/branch/fast-forward/pypy/rlib/libffi.py Wed Dec 1 18:08:21 2010 @@ -178,6 +178,9 @@ # the optimizer will fail to recognize the pattern and won't turn it # into a fast CALL. Note that "arg = arg.next" is optimized away, # assuming that archain is completely virtual. + if argchain.numargs != len(self.argtypes): + raise TypeError, 'Wrong number of arguments: %d expected, got %d' %\ + (argchain.numargs, len(self.argtypes)) ll_args = self._prepare() i = 0 arg = argchain.first Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Wed Dec 1 18:08:21 2010 @@ -185,7 +185,8 @@ return False r_class = rffi.platform.numbertype_to_rclass[tp] assert issubclass(r_class, base_int) - return r_class.BITS < LONG_BIT + return r_class.BITS < LONG_BIT or ( + r_class.BITS == LONG_BIT and r_class.SIGNED) _should_widen_type._annspecialcase_ = 'specialize:memo' del _bits, _itest, _Ltest @@ -486,6 +487,11 @@ r_longlong = build_int('r_longlong', True, 64) r_ulonglong = build_int('r_ulonglong', False, 64) +if r_longlong is not r_int: + r_int64 = r_longlong +else: + r_int64 = int + # float as string -> sign, beforept, afterpt, exponent Modified: pypy/branch/fast-forward/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rdynload.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rdynload.py Wed Dec 1 18:08:21 2010 @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD_7 = platform.name == "freebsd7" +_FREEBSD = platform.name == "freebsd" if _WIN32: from pypy.rlib import rwin32 @@ -27,7 +27,7 @@ else: pre_include_bits = [] -if _FREEBSD_7 or _WIN32: +if _FREEBSD or _WIN32: libraries = [] else: libraries = ['dl'] Modified: pypy/branch/fast-forward/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rsre/rsre_core.py Wed Dec 1 18:08:21 2010 @@ -1,5 +1,5 @@ import sys -from pypy.rlib.debug import check_nonneg, list_not_modified_any_more +from pypy.rlib.debug import check_nonneg from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rsre import rsre_char from pypy.tool.sourcetools import func_with_new_name @@ -91,7 +91,7 @@ # and they must not be more than len(string). check_nonneg(match_start) check_nonneg(end) - self.pattern = list_not_modified_any_more(pattern) + self.pattern = pattern self.match_start = match_start self.end = end self.flags = flags Modified: pypy/branch/fast-forward/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/test/test_libffi.py (original) +++ pypy/branch/fast-forward/pypy/rlib/test/test_libffi.py Wed Dec 1 18:08:21 2010 @@ -262,3 +262,24 @@ # res = self.call(get_dummy, [], rffi.LONG) assert res == initval+1 + + def test_wrong_number_of_arguments(self): + from pypy.rpython.llinterp import LLException + libfoo = self.get_libfoo() + func = (libfoo, 'sum_xy', [types.sint, types.double], types.sint) + + glob = globals() + loc = locals() + def my_raises(s): + try: + exec s in glob, loc + except TypeError: + pass + except LLException, e: + if str(e) != "": + raise + else: + assert False, 'Did not raise' + + my_raises("self.call(func, [38], rffi.LONG)") # one less + my_raises("self.call(func, [38, 12.3, 42], rffi.LONG)") # one more Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll2ctypes.py Wed Dec 1 18:08:21 2010 @@ -446,6 +446,9 @@ "not allocated from RPython at all") self._storage = None + def _getid(self): + return self._addressof_storage() + def __eq__(self, other): if isinstance(other, _llgcopaque): addressof_other = other.intval @@ -1299,7 +1302,7 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7'): + elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/llmemory.py Wed Dec 1 18:08:21 2010 @@ -93,8 +93,10 @@ return endmarker._as_ptr() else: return parent.getitem(index)._as_ptr() - elif (isinstance(A, lltype.FixedSizeArray) and - array_item_type_match(A.OF, self.TYPE)): + elif ((isinstance(A, lltype.FixedSizeArray) + or (isinstance(A, lltype.Array) and A._hints.get('nolength', + False))) + and array_item_type_match(A.OF, self.TYPE)): # for array of primitives or pointers return lltype.direct_ptradd(firstitemptr, self.repeat) else: Modified: pypy/branch/fast-forward/pypy/rpython/memory/gc/inspector.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gc/inspector.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gc/inspector.py Wed Dec 1 18:08:21 2010 @@ -101,7 +101,7 @@ AddressStack = get_address_stack() -class HeapDumper: +class HeapDumper(object): _alloc_flavor_ = "raw" BUFSIZE = 8192 # words Modified: pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_direct.py Wed Dec 1 18:08:21 2010 @@ -60,7 +60,7 @@ pass -class DirectGCTest(object): +class BaseDirectGCTest(object): GC_PARAMS = {} def setup_method(self, meth): @@ -106,6 +106,9 @@ addr = self.gc.malloc(self.get_type_id(TYPE), n, zero=True) return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) + +class DirectGCTest(BaseDirectGCTest): + def test_simple(self): p = self.malloc(S) p.x = 5 Modified: pypy/branch/fast-forward/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gctransform/asmgcroot.py Wed Dec 1 18:08:21 2010 @@ -139,12 +139,13 @@ self._shape_decompressor = ShapeDecompressor() if hasattr(gctransformer.translator, '_jit2gc'): jit2gc = gctransformer.translator._jit2gc - self._extra_gcmapstart = jit2gc['gcmapstart'] - self._extra_gcmapend = jit2gc['gcmapend'] + self._extra_gcmapstart = jit2gc['gcmapstart'] + self._extra_gcmapend = jit2gc['gcmapend'] + self._extra_mark_sorted = jit2gc['gcmarksorted'] else: - returns_null = lambda: llmemory.NULL - self._extra_gcmapstart = returns_null - self._extra_gcmapend = returns_null + self._extra_gcmapstart = lambda: llmemory.NULL + self._extra_gcmapend = lambda: llmemory.NULL + self._extra_mark_sorted = lambda: True def need_thread_support(self, gctransformer, getfn): # Threads supported "out of the box" by the rest of the code. @@ -295,14 +296,16 @@ # we have a non-empty JIT-produced table to look in item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) if item: - self._shape_decompressor.setaddr(item.address[1]) + self._shape_decompressor.setaddr(item) return # maybe the JIT-produced table is not sorted? - sort_gcmap(gcmapstart2, gcmapend2) - item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) - if item: - self._shape_decompressor.setaddr(item.address[1]) - return + was_already_sorted = self._extra_mark_sorted() + if not was_already_sorted: + sort_gcmap(gcmapstart2, gcmapend2) + item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) + if item: + self._shape_decompressor.setaddr(item) + return # the item may have been not found because the main array was # not sorted. Sort it and try again. win32_follow_gcmap_jmp(gcmapstart, gcmapend) @@ -357,7 +360,8 @@ The interval from the start address (included) to the end address (excluded) is assumed to be a sorted arrays of pairs (addr1, addr2). This searches for the item with a given addr1 and returns its - address. + address. If not found exactly, it tries to return the address + of the item left of addr1 (i.e. such that result.address[0] < addr1). """ count = (end - start) // arrayitemsize while count > 1: @@ -386,7 +390,7 @@ # (item.signed[1] is an address in this case, not a signed at all!) item = binary_search(gcmapstart, gcmapend, retaddr) if item.address[0] == retaddr: - return item # found + return item.address[1] # found else: return llmemory.NULL # failed Modified: pypy/branch/fast-forward/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/fast-forward/pypy/rpython/module/ll_os.py Wed Dec 1 18:08:21 2010 @@ -1386,6 +1386,25 @@ return extdef([], (int, int), "ll_os.ll_os_openpty", llimpl=openpty_llimpl) + @registering_if(os, 'forkpty') + def register_os_forkpty(self): + os_forkpty = self.llexternal( + 'forkpty', + [rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP], + rffi.PID_T, + compilation_info=ExternalCompilationInfo(libraries=['util'])) + def forkpty_llimpl(): + master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + childpid = os_forkpty(master_p, None, None, None) + master_fd = master_p[0] + lltype.free(master_p, flavor='raw') + if childpid == -1: + raise OSError(rposix.get_errno(), "os_forkpty failed") + return (rffi.cast(lltype.Signed, childpid), + rffi.cast(lltype.Signed, master_fd)) + + return extdef([], (int, int), "ll_os.ll_os_forkpty", + llimpl=forkpty_llimpl) @registering(os._exit) def register_os__exit(self): Modified: pypy/branch/fast-forward/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/module/ll_time.py (original) +++ pypy/branch/fast-forward/pypy/rpython/module/ll_time.py Wed Dec 1 18:08:21 2010 @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if sys.platform == 'freebsd7': +if "freebsd" in sys.platform: libraries = ['compat'] else: libraries = [] Modified: pypy/branch/fast-forward/pypy/rpython/rlist.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/rlist.py (original) +++ pypy/branch/fast-forward/pypy/rpython/rlist.py Wed Dec 1 18:08:21 2010 @@ -9,7 +9,7 @@ from pypy.rpython import robject from pypy.rlib.objectmodel import malloc_zero_filled from pypy.rlib.debug import ll_assert -from pypy.rlib.rarithmetic import ovfcheck, widen +from pypy.rlib.rarithmetic import ovfcheck, widen, r_uint, intmask from pypy.rpython.annlowlevel import ADTInterface from pypy.rlib import rgc @@ -241,17 +241,22 @@ class __extend__(pairtype(AbstractBaseListRepr, IntegerRepr)): def rtype_getitem((r_lst, r_int), hop, checkidx=False): + v_lst, v_index = hop.inputargs(r_lst, Signed) if checkidx: - spec = dum_checkidx + hop.exception_is_here() else: - spec = dum_nocheck - v_func = hop.inputconst(Void, spec) - v_lst, v_index = hop.inputargs(r_lst, Signed) + hop.exception_cannot_occur() if hop.args_s[0].listdef.listitem.mutated or checkidx: if hop.args_s[1].nonneg: llfn = ll_getitem_nonneg else: llfn = ll_getitem + if checkidx: + spec = dum_checkidx + else: + spec = dum_nocheck + c_func_marker = hop.inputconst(Void, spec) + v_res = hop.gendirectcall(llfn, c_func_marker, v_lst, v_index) else: # this is the 'foldable' version, which is not used when # we check for IndexError @@ -259,11 +264,7 @@ llfn = ll_getitem_foldable_nonneg else: llfn = ll_getitem_foldable - if checkidx: - hop.exception_is_here() - else: - hop.exception_cannot_occur() - v_res = hop.gendirectcall(llfn, v_func, v_lst, v_index) + v_res = hop.gendirectcall(llfn, v_lst, v_index) return r_lst.recast(hop.llops, v_res) rtype_getitem_key = rtype_getitem @@ -538,12 +539,14 @@ dest.ll_setitem_fast(dest_start + i, item) i += 1 ll_arraycopy._annenforceargs_ = [None, None, int, int, int] +# no oopspec -- the function is inlined by the JIT def ll_copy(RESLIST, l): length = l.ll_length() new_lst = RESLIST.ll_newlist(length) ll_arraycopy(l, new_lst, 0, 0, length) return new_lst +# no oopspec -- the function is inlined by the JIT def ll_len(l): return l.ll_length() @@ -551,6 +554,7 @@ def ll_list_is_true(l): # check if a list is True, allowing for None return bool(l) and l.ll_length() != 0 +# no oopspec -- the function is inlined by the JIT def ll_len_foldable(l): return l.ll_length() @@ -558,6 +562,7 @@ def ll_list_is_true_foldable(l): return bool(l) and ll_len_foldable(l) != 0 +# no oopspec -- the function is inlined by the JIT def ll_append(l, newitem): length = l.ll_length() @@ -588,6 +593,7 @@ ll_arraycopy(l1, l, 0, 0, len1) ll_arraycopy(l2, l, 0, len1, len2) return l +# no oopspec -- the function is inlined by the JIT def ll_insert_nonneg(l, index, newitem): length = l.ll_length() @@ -674,60 +680,72 @@ l.ll_setitem_fast(length_1_i, tmp) i += 1 length_1_i -= 1 +ll_reverse.oopspec = 'list.reverse(l)' def ll_getitem_nonneg(func, l, index): ll_assert(index >= 0, "unexpectedly negative list getitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError - else: - ll_assert(index < l.ll_length(), "list getitem index out of bound") return l.ll_getitem_fast(index) -ll_getitem_nonneg.oopspec = 'list.getitem(l, index)' +ll_getitem_nonneg._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_getitem(func, l, index): - length = l.ll_length() - if index < 0: - index += length if func is dum_checkidx: - if index < 0 or index >= length: - raise IndexError + length = l.ll_length() # common case: 0 <= index < length + if r_uint(index) >= r_uint(length): + # Failed, so either (-length <= index < 0), or we have to raise + # IndexError. First add 'length' to get the final index, then + # check that we now have (0 <= index < length). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(index >= 0, "negative list getitem index out of bound") - ll_assert(index < length, "list getitem index out of bound") + # We don't want checking, but still want to support index < 0. + # Only call ll_length() if needed. + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list getitem index out of bound") return l.ll_getitem_fast(index) -ll_getitem.oopspec = 'list.getitem(l, index)' +# no oopspec -- the function is inlined by the JIT -def ll_getitem_foldable_nonneg(func, l, index): - return ll_getitem_nonneg(func, l, index) +def ll_getitem_foldable_nonneg(l, index): + ll_assert(index >= 0, "unexpectedly negative list getitem index") + return l.ll_getitem_fast(index) ll_getitem_foldable_nonneg.oopspec = 'list.getitem_foldable(l, index)' -def ll_getitem_foldable(func, l, index): - return ll_getitem(func, l, index) -ll_getitem_foldable.oopspec = 'list.getitem_foldable(l, index)' +def ll_getitem_foldable(l, index): + if index < 0: + index += l.ll_length() + return ll_getitem_foldable_nonneg(l, index) +ll_getitem_foldable._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_setitem_nonneg(func, l, index, newitem): ll_assert(index >= 0, "unexpectedly negative list setitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError - else: - ll_assert(index < l.ll_length(), "list setitem index out of bound") l.ll_setitem_fast(index, newitem) -ll_setitem_nonneg.oopspec = 'list.setitem(l, index, newitem)' +ll_setitem_nonneg._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_setitem(func, l, index, newitem): - length = l.ll_length() - if index < 0: - index += length if func is dum_checkidx: - if index < 0 or index >= length: - raise IndexError + length = l.ll_length() + if r_uint(index) >= r_uint(length): # see comments in ll_getitem(). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(index >= 0, "negative list setitem index out of bound") - ll_assert(index < length, "list setitem index out of bound") + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list setitem index out of bound") l.ll_setitem_fast(index, newitem) -ll_setitem.oopspec = 'list.setitem(l, index, newitem)' +# no oopspec -- the function is inlined by the JIT def ll_delitem_nonneg(func, l, index): ll_assert(index >= 0, "unexpectedly negative list delitem index") @@ -751,19 +769,20 @@ l._ll_resize_le(newlength) ll_delitem_nonneg.oopspec = 'list.delitem(l, index)' -def ll_delitem(func, l, i): - length = l.ll_length() - if i < 0: - i += length +def ll_delitem(func, l, index): if func is dum_checkidx: - if i < 0 or i >= length: - raise IndexError + length = l.ll_length() + if r_uint(index) >= r_uint(length): # see comments in ll_getitem(). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(i >= 0, "negative list delitem index out of bound") - ll_assert(i < length, "list delitem index out of bound") - ll_delitem_nonneg(dum_nocheck, l, i) -ll_delitem.oopspec = 'list.delitem(l, i)' - + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list delitem index out of bound") + ll_delitem_nonneg(dum_nocheck, l, index) +# no oopspec -- the function is inlined by the JIT def ll_extend(l1, l2): len1 = l1.ll_length() @@ -799,6 +818,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_str_slice_startstop(lst, s, getstrlen, getstritem, start, stop): @@ -824,6 +844,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_str_slice_minusone(lst, s, getstrlen, getstritem): len1 = lst.ll_length() @@ -843,6 +864,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_char_count(lst, char, count): if count <= 0: @@ -859,6 +881,7 @@ while j < newlength: lst.ll_setitem_fast(j, char) j += 1 +# not inlined by the JIT -- contains a loop def ll_listslice_startonly(RESLIST, l1, start): len1 = l1.ll_length() @@ -869,6 +892,7 @@ ll_arraycopy(l1, l, start, 0, newlength) return l ll_listslice_startonly._annenforceargs_ = (None, None, int) +# no oopspec -- the function is inlined by the JIT def ll_listslice_startstop(RESLIST, l1, start, stop): length = l1.ll_length() @@ -881,6 +905,7 @@ l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, start, 0, newlength) return l +# no oopspec -- the function is inlined by the JIT def ll_listslice_minusone(RESLIST, l1): newlength = l1.ll_length() - 1 @@ -888,6 +913,7 @@ l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, 0, 0, newlength) return l +# no oopspec -- the function is inlined by the JIT def ll_listdelslice_startonly(l, start): ll_assert(start >= 0, "del l[start:] with unexpectedly negative start") @@ -958,6 +984,7 @@ return False j += 1 return True +# not inlined by the JIT -- contains a loop def ll_listcontains(lst, obj, eqfn): lng = lst.ll_length() @@ -971,6 +998,7 @@ return True j += 1 return False +# not inlined by the JIT -- contains a loop def ll_listindex(lst, obj, eqfn): lng = lst.ll_length() @@ -984,6 +1012,7 @@ return j j += 1 raise ValueError # can't say 'list.index(x): x not in list' +# not inlined by the JIT -- contains a loop def ll_listremove(lst, obj, eqfn): index = ll_listindex(lst, obj, eqfn) # raises ValueError if obj not in lst @@ -1030,3 +1059,4 @@ i += 1 j += length return res +# not inlined by the JIT -- contains a loop Modified: pypy/branch/fast-forward/pypy/rpython/test/test_rint.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/test/test_rint.py (original) +++ pypy/branch/fast-forward/pypy/rpython/test/test_rint.py Wed Dec 1 18:08:21 2010 @@ -4,15 +4,10 @@ from pypy.annotation import model as annmodel from pypy.rpython.test import snippet from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong -from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.rarithmetic import ovfcheck, r_int64 from pypy.rlib import objectmodel from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -if r_longlong is not r_int: - int64 = r_longlong -else: - int64 = int - class TestSnippet(object): @@ -111,10 +106,10 @@ def f(i): return str(i) - res = self.interpret(f, [int64(0)]) + res = self.interpret(f, [r_int64(0)]) assert self.ll_to_string(res) == '0' - res = self.interpret(f, [int64(413974738222117)]) + res = self.interpret(f, [r_int64(413974738222117)]) assert self.ll_to_string(res) == '413974738222117' def test_unsigned(self): @@ -136,7 +131,7 @@ f._annspecialcase_ = "specialize:argtype(0)" def g(n): if n > 0: - return f(int64(0)) + return f(r_int64(0)) else: return f(0) res = self.interpret(g, [0]) @@ -148,7 +143,7 @@ def test_downcast_int(self): def f(i): return int(i) - res = self.interpret(f, [int64(0)]) + res = self.interpret(f, [r_int64(0)]) assert res == 0 def test_isinstance_vs_int_types(self): @@ -158,7 +153,7 @@ return [None] if isinstance(x, str): return x - if isinstance(x, int64): + if isinstance(x, r_int64): return int(x) return "XXX" wrap._annspecialcase_ = 'specialize:argtype(0)' @@ -166,7 +161,7 @@ space = FakeSpace() def wrap(x): return space.wrap(x) - res = self.interpret(wrap, [int64(0)]) + res = self.interpret(wrap, [r_int64(0)]) assert res == 0 def test_truediv(self): @@ -179,25 +174,25 @@ def test_float_conversion(self): def f(ii): return float(ii) - res = self.interpret(f, [int64(100000000)]) + res = self.interpret(f, [r_int64(100000000)]) assert type(res) is float assert res == 100000000. - res = self.interpret(f, [int64(1234567890123456789)]) + res = self.interpret(f, [r_int64(1234567890123456789)]) assert type(res) is float assert self.float_eq(res, 1.2345678901234568e+18) def test_float_conversion_implicit(self): def f(ii): return 1.0 + ii - res = self.interpret(f, [int64(100000000)]) + res = self.interpret(f, [r_int64(100000000)]) assert type(res) is float assert res == 100000001. - res = self.interpret(f, [int64(1234567890123456789)]) + res = self.interpret(f, [r_int64(1234567890123456789)]) assert type(res) is float assert self.float_eq(res, 1.2345678901234568e+18) def test_rarithmetic(self): - inttypes = [int, r_uint, int64, r_ulonglong] + inttypes = [int, r_uint, r_int64, r_ulonglong] for inttype in inttypes: c = inttype() def f(): @@ -232,16 +227,16 @@ res = self.interpret(f, [int(-1<<(r_int.BITS-1))]) assert res == 0 - res = self.interpret(f, [int64(-1)]) + res = self.interpret(f, [r_int64(-1)]) assert res == 1 - res = self.interpret(f, [int64(-1)<<(r_longlong.BITS-1)]) + res = self.interpret(f, [r_int64(-1)<<(r_longlong.BITS-1)]) assert res == 0 div_mod_iteration_count = 1000 def test_div_mod(self): import random - for inttype in (int, int64): + for inttype in (int, r_int64): def d(x, y): return x/y @@ -304,7 +299,7 @@ except ZeroDivisionError: return 84 - for inttype in (int, int64): + for inttype in (int, r_int64): args = [( 5, 2), (-5, 2), ( 5,-2), (-5,-2), ( 6, 2), (-6, 2), ( 6,-2), (-6,-2), Modified: pypy/branch/fast-forward/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/test/test_rlist.py (original) +++ pypy/branch/fast-forward/pypy/rpython/test/test_rlist.py Wed Dec 1 18:08:21 2010 @@ -12,6 +12,7 @@ from pypy.rpython.rint import signed_repr from pypy.objspace.flow.model import Constant, Variable from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +from pypy.rlib.debug import ll_assert # undo the specialization parameter for n1 in 'get set del'.split(): @@ -1076,7 +1077,13 @@ res = self.interpret(f, [0]) assert res == 1 - py.test.raises(AssertionError, self.interpret, f, [1]) + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) + else: + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def f(x): l = [1] @@ -1121,12 +1128,13 @@ res = self.interpret(f, [0]) assert res == 1 - try: - self.interpret_raises(IndexError, f, [1]) - except (AssertionError,), e: - pass + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) else: - assert False + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def f(x): l = [1] @@ -1163,12 +1171,13 @@ res = self.interpret(f, [0]) assert res == 1 - try: - self.interpret_raises(IndexError, f, [1]) - except (AssertionError,), e: - pass + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) else: - assert False + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def test_charlist_extension_1(self): def f(n): @@ -1327,8 +1336,32 @@ res = self.interpret(f, [2]) assert res == True + def test_immutable_list_out_of_instance(self): + from pypy.translator.simplify import get_funcobj + for immutable_fields in (["a", "b"], ["a", "b", "y[*]"]): + class A(object): + _immutable_fields_ = immutable_fields + class B(A): + pass + def f(i): + b = B() + lst = [i] + lst[0] += 1 + b.y = lst + ll_assert(b.y is lst, "copying when reading out the attr?") + return b.y[0] + res = self.interpret(f, [10]) + assert res == 11 + t, rtyper, graph = self.gengraph(f, [int]) + block = graph.startblock + op = block.operations[-1] + assert op.opname == 'direct_call' + func = get_funcobj(op.args[0].value)._callable + assert ('foldable' in func.func_name) == \ + ("y[*]" in immutable_fields) class TestLLtype(BaseTestRlist, LLRtypeMixin): + type_system = 'lltype' rlist = ll_rlist def test_memoryerror(self): @@ -1420,14 +1453,16 @@ lst2 = [i] lst2.append(42) # mutated list return lst1[i] + lst2[i] - _, _, graph = self.gengraph(f, [int]) + from pypy.annotation import model as annmodel + _, _, graph = self.gengraph(f, [annmodel.SomeInteger(nonneg=True)]) block = graph.startblock lst1_getitem_op = block.operations[-3] # XXX graph fishing lst2_getitem_op = block.operations[-2] func1 = lst1_getitem_op.args[0].value._obj._callable func2 = lst2_getitem_op.args[0].value._obj._callable assert func1.oopspec == 'list.getitem_foldable(l, index)' - assert func2.oopspec == 'list.getitem(l, index)' + assert not hasattr(func2, 'oopspec') class TestOOtype(BaseTestRlist, OORtypeMixin): rlist = oo_rlist + type_system = 'ootype' Modified: pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py Wed Dec 1 18:08:21 2010 @@ -673,6 +673,7 @@ C_HEADER = """ #include #include /* for offsetof() */ +#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); Modified: pypy/branch/fast-forward/pypy/tool/ansi_print.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/ansi_print.py (original) +++ pypy/branch/fast-forward/pypy/tool/ansi_print.py Wed Dec 1 18:08:21 2010 @@ -16,6 +16,7 @@ 'WARNING': ((31,), False), 'event': ((1,), True), 'ERROR': ((1, 31), False), + 'Error': ((1, 31), False), 'info': ((35,), False), 'stub': ((34,), False), } Modified: pypy/branch/fast-forward/pypy/tool/logparser.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/logparser.py (original) +++ pypy/branch/fast-forward/pypy/tool/logparser.py Wed Dec 1 18:08:21 2010 @@ -25,7 +25,7 @@ return parse_log(lines, verbose=verbose) def parse_log(lines, verbose=False): - color = "(?:\x1b.*?m)" + color = "(?:\x1b.*?m)?" r_start = re.compile(color + r"\[([0-9a-fA-F]+)\] \{([\w-]+)" + color + "$") r_stop = re.compile(color + r"\[([0-9a-fA-F]+)\] ([\w-]+)\}" + color + "$") lasttime = 0 Modified: pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/gcc/trackgcroot.py Wed Dec 1 18:08:21 2010 @@ -1106,7 +1106,7 @@ format = 'darwin64' function_names_prefix = '_' - LABEL = ElfFunctionGcRootTracker32.LABEL + LABEL = ElfFunctionGcRootTracker64.LABEL r_jmptable_item = re.compile(r"\t.(?:long|quad)\t"+LABEL+"(-\"?[A-Za-z0-9$]+\"?)?\s*$") r_functionstart = re.compile(r"_(\w+):\s*$") Modified: pypy/branch/fast-forward/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/src/mem.h (original) +++ pypy/branch/fast-forward/pypy/translator/c/src/mem.h Wed Dec 1 18:08:21 2010 @@ -233,4 +233,4 @@ #define OP_GC_GET_RPY_MEMORY_USAGE(x, r) r = -1 #define OP_GC_GET_RPY_TYPE_INDEX(x, r) r = -1 #define OP_GC_IS_RPY_INSTANCE(x, r) r = 0 -#define OP_GC_DUMP_RPY_HEAP(r) r = 0 +#define OP_GC_DUMP_RPY_HEAP(fd, r) r = 0 Modified: pypy/branch/fast-forward/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/goal/app_main.py (original) +++ pypy/branch/fast-forward/pypy/translator/goal/app_main.py Wed Dec 1 18:08:21 2010 @@ -11,6 +11,7 @@ -h, --help show this help message and exit -m library module to be run as a script (terminates option list) -W arg warning control (arg is action:message:category:module:lineno) + -E ignore environment variables (such as PYTHONPATH) --version print the PyPy version --info print translation information about this PyPy executable """ @@ -204,7 +205,7 @@ break # found! return newpath -def setup_initial_paths(executable, nanos): +def setup_initial_paths(executable, nanos, readenv=True, **extra): # a substituted os if we are translated global os os = nanos @@ -225,7 +226,7 @@ sys.executable = os.path.abspath(executable) newpath = get_library_path(executable) - path = os.getenv('PYTHONPATH') + path = readenv and os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath # remove duplicates @@ -235,7 +236,6 @@ if dir not in _seen: sys.path.append(dir) _seen[dir] = True - return executable # Order is significant! sys_flags = ( @@ -271,6 +271,7 @@ options['warnoptions'] = [] print_sys_flags = False i = 0 + readenv = True while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -310,6 +311,8 @@ argv[i] = '-c' options["run_command"] = True break + elif arg == '-E': + readenv = False elif arg == '-u': options["unbuffered"] = True elif arg == '-O' or arg == '-OO': @@ -378,6 +381,7 @@ run_stdin, warnoptions, unbuffered, + readenv, cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 @@ -431,7 +435,7 @@ # * PYTHONINSPECT is set and stdin is a tty. # return (interactive or - (os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) + (readenv and os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) success = True @@ -464,7 +468,7 @@ # If stdin is a tty or if "-i" is specified, we print # a banner and run $PYTHONSTARTUP. print_banner() - python_startup = os.getenv('PYTHONSTARTUP') + python_startup = readenv and os.getenv('PYTHONSTARTUP') if python_startup: try: f = open(python_startup) @@ -538,7 +542,6 @@ '"license" for more information.') def entry_point(executable, argv, nanos): - executable = setup_initial_paths(executable, nanos) try: cmdline = parse_command_line(argv) except CommandLineError, e: @@ -546,8 +549,8 @@ return 2 if cmdline is None: return 0 - else: - return run_command_line(**cmdline) + setup_initial_paths(executable, nanos, **cmdline) + return run_command_line(**cmdline) if __name__ == '__main__': @@ -582,13 +585,13 @@ sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path os = nanos.os_module_for_testing - sys.ps1 = '>>>> ' - sys.ps2 = '.... ' try: sys.exit(int(entry_point(sys.argv[0], sys.argv[1:], os))) finally: - sys.ps1 = '>>> ' # restore the normal ones, in case - sys.ps2 = '... ' # we are dropping to CPython's prompt + # restore the normal prompt (which was changed by _pypy_interact), in + # case we are dropping to CPython's prompt + sys.ps1 = '>>> ' + sys.ps2 = '... ' import os; os.environ.update(reset) assert old_argv is sys.argv assert old_path is sys.path Modified: pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py Wed Dec 1 18:08:21 2010 @@ -217,6 +217,38 @@ finally: os.environ['PYTHONSTARTUP'] = old + def test_ignore_python_startup(self): + old = os.environ.get('PYTHONSTARTUP', '') + try: + os.environ['PYTHONSTARTUP'] = crashing_demo_script + child = self.spawn(['-E']) + child.expect(re.escape(banner)) + index = child.expect(['Traceback', '>>> ']) + assert index == 1 # no traceback + finally: + os.environ['PYTHONSTARTUP'] = old + + def test_ignore_python_inspect(self): + os.environ['PYTHONINSPECT_'] = '1' + try: + child = self.spawn(['-E', '-c', 'pass']) + from pexpect import EOF + index = child.expect(['>>> ', EOF]) + assert index == 1 # no prompt + finally: + del os.environ['PYTHONINSPECT_'] + + def test_ignore_python_path(self): + old = os.environ.get('PYTHONPATH', '') + try: + os.environ['PYTHONPATH'] = 'foobarbaz' + child = self.spawn(['-E', '-c', 'import sys; print sys.path']) + from pexpect import EOF + index = child.expect(['foobarbaz', EOF]) + assert index == 1 # no foobarbaz + finally: + os.environ['PYTHONPATH'] = old + def test_unbuffered(self): line = 'import os,sys;sys.stdout.write(str(789));os.read(0,1)' child = self.spawn(['-u', '-c', line]) @@ -334,6 +366,10 @@ child = self.spawn(['-mpypy.translator.goal.test2.mymodule']) child.expect('mymodule running') + def test_ps1_only_if_interactive(self): + argv = ['-c', 'import sys; print hasattr(sys, "ps1")'] + child = self.spawn(argv) + child.expect('False') class TestNonInteractive: Modified: pypy/branch/fast-forward/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/platform/__init__.py (original) +++ pypy/branch/fast-forward/pypy/translator/platform/__init__.py Wed Dec 1 18:08:21 2010 @@ -123,7 +123,9 @@ errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() for line in stderrlines: - log.ERROR(line) + log.Error(line) + # ^^^ don't use ERROR, because it might actually be fine. + # Also, ERROR confuses lib-python/conftest.py. raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): @@ -215,13 +217,13 @@ host_factory = Darwin_i386 else: host_factory = Darwin_x86_64 -elif sys.platform == 'freebsd7': - from pypy.translator.platform.freebsd7 import Freebsd7, Freebsd7_64 +elif "freebsd" in sys.platform: + from pypy.translator.platform.freebsd import Freebsd, Freebsd_64 import platform if platform.architecture()[0] == '32bit': - host_factory = Freebsd7 + host_factory = Freebsd else: - host_factory = Freebsd7_64 + host_factory = Freebsd_64 elif os.name == 'nt': from pypy.translator.platform.windows import Windows host_factory = Windows Modified: pypy/branch/fast-forward/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/platform/darwin.py (original) +++ pypy/branch/fast-forward/pypy/translator/platform/darwin.py Wed Dec 1 18:08:21 2010 @@ -12,12 +12,14 @@ so_ext = 'so' + default_cc = 'gcc' + def __init__(self, cc=None): if cc is None: try: cc = os.environ['CC'] except KeyError: - cc = 'gcc' + cc = self.default_cc self.cc = cc def _args_for_shared(self, args): @@ -85,3 +87,4 @@ link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4') cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4') + default_cc = 'gcc-4.0' Modified: pypy/branch/fast-forward/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/fast-forward/pypy/translator/tool/cbuild.py Wed Dec 1 18:08:21 2010 @@ -309,6 +309,7 @@ #define _POSIX_C_SOURCE 200112L /* Define on FreeBSD to activate all library features */ #define __BSD_VISIBLE 1 +#define __XSI_VISIBLE 700 /* Windows: winsock/winsock2 mess */ #define WIN32_LEAN_AND_MEAN ''' From david at codespeak.net Wed Dec 1 18:33:15 2010 From: david at codespeak.net (david at codespeak.net) Date: Wed, 1 Dec 2010 18:33:15 +0100 (CET) Subject: [pypy-svn] r79721 - in pypy/branch/arm-backend/pypy/jit/backend: . arm llsupport Message-ID: <20101201173315.4D09B282BEC@codespeak.net> Author: david Date: Wed Dec 1 18:33:12 2010 New Revision: 79721 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py pypy/branch/arm-backend/pypy/jit/backend/arm/runner.py pypy/branch/arm-backend/pypy/jit/backend/llsupport/llmodel.py pypy/branch/arm-backend/pypy/jit/backend/model.py Log: Rpythonification and some bugfixes Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Wed Dec 1 18:33:12 2010 @@ -5,7 +5,7 @@ from pypy.jit.backend.arm.codebuilder import ARMv7Builder, ARMv7InMemoryBuilder from pypy.jit.backend.arm.regalloc import ARMRegisterManager, ARMFrameManager from pypy.jit.backend.llsupport.regalloc import compute_vars_longevity, TempBox -from pypy.jit.metainterp.history import (Const, ConstInt, BoxInt, BasicFailDescr, +from pypy.jit.metainterp.history import (Const, ConstInt, BoxInt, AbstractFailDescr, INT, REF, FLOAT) from pypy.jit.metainterp.resoperation import rop from pypy.rlib import rgc @@ -26,6 +26,7 @@ self.cpu = cpu self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) + self.setup_failure_recovery() self._debug_asm = True self.mc = None self.malloc_func_addr = 0 @@ -74,6 +75,8 @@ self.failure_recovery_func = failure_recovery_func + recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Signed, lltype.Signed], lltype.Signed)) + @rgc.no_collect def decode_registers_and_descr(self, mem_loc, frame_loc, regs_loc): """Decode locations encoded in memory at mem_loc and write the values to @@ -169,9 +172,7 @@ mem[i+3] = chr((n >> 24) & 0xFF) def _gen_exit_path(self): - self.setup_failure_recovery() - functype = lltype.Ptr(lltype.FuncType([lltype.Signed, lltype.Signed, lltype.Signed], lltype.Signed)) - decode_registers_addr = llhelper(functype, self.failure_recovery_func) + decode_registers_addr = llhelper(self.recovery_func_sign, self.failure_recovery_func) self.mc.PUSH([reg.value for reg in r.all_regs]) # registers r0 .. r10 self.mc.MOV_rr(r.r0.value, r.lr.value) # move mem block address, to r0 to pass as @@ -199,6 +200,7 @@ descr = op.getdescr() if op.getopnum() != rop.FINISH: + assert isinstance(descr, AbstractFailDescr) descr._arm_frame_depth = regalloc.frame_manager.frame_depth reg = r.lr # XXX free this memory @@ -274,7 +276,7 @@ raise ValueError self.mc.gen_load_int(reg.value, addr) self.mc.LDR_ri(reg.value, reg.value) - regalloc.possibly_free_var(reg) + regalloc.possibly_free_var(loc) arglocs = [regalloc.loc(arg) for arg in inputargs] looptoken._arm_arglocs = arglocs return arglocs @@ -368,19 +370,17 @@ rev = False if n <= 0xFF and fcond == c.AL: if rev: - op = cb.ADD_ri + cb.ADD_ri(r.sp.value, base_reg.value, n) else: - op = cb.SUB_ri - op(r.sp.value, base_reg.value, n) + cb.SUB_ri(r.sp.value, base_reg.value, n) else: b = TempBox() reg = regalloc.force_allocate_reg(b) cb.gen_load_int(reg.value, n, cond=fcond) if rev: - op = cb.ADD_rr + cb.ADD_rr(r.sp.value, base_reg.value, reg.value, cond=fcond) else: - op = cb.SUB_rr - op(r.sp.value, base_reg.value, reg.value, cond=fcond) + cb.SUB_rr(r.sp.value, base_reg.value, reg.value, cond=fcond) regalloc.possibly_free_var(b) def _walk_operations(self, operations, regalloc): Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py Wed Dec 1 18:33:12 2010 @@ -7,7 +7,9 @@ from pypy.rlib.rmmap import alloc, PTR from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import lltype, rffi -from pypy.jit.metainterp.history import ConstInt, BoxInt, BasicFailDescr +from pypy.jit.metainterp.history import ConstInt, BoxInt, AbstractFailDescr +from pypy.rlib.objectmodel import we_are_translated +from pypy.tool.udir import udir def binary_helper_call(name): signature = getattr(arch, 'arm_%s_sign' % name) @@ -20,7 +22,7 @@ self.BL(addr) else: self.PUSH(range(2, 4), cond=c) - self.BL(addr, cond=c, some_reg=reg.r2) + self.BL(addr, c, some_reg=reg.r2) self.POP(range(2,4), cond=c) return f @@ -30,14 +32,18 @@ self._size = map_size self._pos = 0 - def _dump_trace(self, name): - f = open('output/%s' % name, 'wb') - for i in range(self._pos): - f.write(self._data[i]) - f.close() + def _dump_trace(self, name, formatter=-1): + if not we_are_translated(): + if formatter != -1: + name = name % formatter + dir = udir.ensure('asm', dir=True) + f = dir.join(name).open('wb') + for i in range(self._pos): + f.write(self._data[i]) + f.close() def ensure_can_fit(self, n): - raise NotImplentedError + raise NotImplementedError def NOP(self): self.MOV_rr(0, 0) @@ -163,7 +169,7 @@ new_mem_addr = rffi.cast(lltype.Signed, new_mem) self.LDR_ri(reg.pc.value, reg.pc.value, -4) self.write32(new_mem_addr) - self._dump_trace('data%04d.asm' % self.n_data) + self._dump_trace('data%04d.asm', self.n_data) self.n_data += 1 self._data = new_mem self._pos = 0 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Wed Dec 1 18:33:12 2010 @@ -17,7 +17,7 @@ from pypy.jit.backend.llsupport.regalloc import compute_vars_longevity, TempBox from pypy.jit.codewriter import heaptracker from pypy.jit.metainterp.history import (Const, ConstInt, BoxInt, Box, - BasicFailDescr, LoopToken, INT, REF) + AbstractFailDescr, LoopToken, INT, FLOAT, REF) from pypy.jit.metainterp.resoperation import rop from pypy.rlib import rgc from pypy.rlib.objectmodel import we_are_translated @@ -169,11 +169,12 @@ #XXX check for a better way of doing this def emit_op_int_neg(self, op, regalloc, fcond): arg = op.getarg(0) - l0 = regalloc.make_sure_var_in_reg(arg, imm_fine=False) + resbox = op.result + l0 = regalloc.make_sure_var_in_reg(arg) l1 = regalloc.make_sure_var_in_reg(ConstInt(-1), [arg], imm_fine=False) - res = regalloc.force_allocate_reg(op.result, [arg]) - self.mc.MUL(res.value, l0.value, l1.value) - regalloc.possibly_free_vars([l0, l1, res]) + resloc = regalloc.force_allocate_reg(resbox, [arg]) + self.mc.MUL(resloc.value, l0.value, l1.value) + regalloc.possibly_free_vars([arg, resbox]) return fcond class GuardOpAssembler(object): @@ -183,7 +184,7 @@ guard_size = ARMv7Builder.size_of_gen_load_int + 6*WORD def _emit_guard(self, op, regalloc, fcond, save_exc=False): descr = op.getdescr() - assert isinstance(descr, BasicFailDescr) + assert isinstance(descr, AbstractFailDescr) #if hasattr(op, 'getfailargs'): # print 'Failargs: ', op.getfailargs() @@ -206,14 +207,14 @@ a0 = op.getarg(0) l0 = regalloc.make_sure_var_in_reg(a0, imm_fine=False) self.mc.CMP_ri(l0.value, 0) - regalloc.possibly_free_var(l0) + regalloc.possibly_free_var(a0) return self._emit_guard(op, regalloc, c.NE) def emit_op_guard_false(self, op, regalloc, fcond): a0 = op.getarg(0) l0 = regalloc.make_sure_var_in_reg(a0, imm_fine=False) self.mc.CMP_ri(l0.value, 0) - regalloc.possibly_free_var(l0) + regalloc.possibly_free_var(a0) return self._emit_guard(op, regalloc, c.EQ) def emit_op_guard_value(self, op, regalloc, fcond): @@ -259,8 +260,8 @@ x = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) y_temp = TempBox() y = regalloc.force_allocate_reg(y_temp) - self.mc.gen_load_int(y.value, - self.cpu.cast_adr_to_int(op.getarg(1).getint())) + arg = op.getarg(1).getint() + self.mc.gen_load_int(y.value, rffi.cast(lltype.Signed, arg)) return [x, y], [op.getarg(0), y_temp] @@ -276,7 +277,7 @@ self.mc.CMP_rr(l0.value, y.value) regalloc.possibly_free_var(t0) else: - raise NotImplentedError + raise NotImplementedError # XXX port from x86 backend once gc support is in place regalloc.possibly_free_vars_for_op(op) @@ -289,11 +290,13 @@ _mixin_ = True def emit_op_jump(self, op, regalloc, fcond): - destlocs = op.getdescr()._arm_arglocs + descr = op.getdescr() + assert isinstance(descr, LoopToken) + destlocs = descr._arm_arglocs srclocs = [regalloc.loc(op.getarg(i)) for i in range(op.numargs())] remap_frame_layout(self, srclocs, destlocs, r.ip) - loop_code = op.getdescr()._arm_loop_code + loop_code = descr._arm_loop_code self.mc.B(loop_code, fcond) return fcond @@ -302,7 +305,7 @@ return fcond def emit_op_call(self, op, regalloc, fcond, save_all_regs=False): - adr = self.cpu.cast_adr_to_int(op.getarg(0).getint()) + adr = rffi.cast(lltype.Signed, op.getarg(0).getint()) args = op.getarglist()[1:] cond = self._emit_call(adr, args, regalloc, fcond, save_all_regs, op.result) @@ -328,7 +331,7 @@ for i in range(4, n_args): reg = regalloc.make_sure_var_in_reg(args[i]) self.mc.STR_ri(reg.value, r.sp.value, (i-4)*WORD) - regalloc.possibly_free_var(reg) + regalloc.possibly_free_var(args[i]) reg_args = min(n_args, 4) @@ -337,10 +340,8 @@ selected_reg=r.all_regs[i]) locs.append(l) # XXX use PUSH here instead of spilling every reg for itself - if save_all_regs: - regalloc.before_call(r.all_regs, save_all_regs) - else: - regalloc.before_call() + regalloc.before_call(save_all_regs=save_all_regs) + self.mc.BL(adr) if result: @@ -375,8 +376,9 @@ self.mc.gen_load_int(loc.value, self.cpu.pos_exception(), fcond) self.mc.LDR_ri(loc.value, loc.value) self.mc.CMP_ri(loc.value, 0) - self._emit_guard(op, regalloc, c.EQ, save_exc=True) + cond = self._emit_guard(op, regalloc, c.EQ, save_exc=True) regalloc.possibly_free_var(t) + return cond def emit_op_guard_exception(self, op, regalloc, fcond): args = op.getarglist() @@ -385,8 +387,8 @@ loc = regalloc.force_allocate_reg(t, args) loc1 = regalloc.force_allocate_reg(t1, args + [t]) self.mc.gen_load_int(loc.value, - self.cpu.cast_adr_to_int( - op.getarg(0).getint()), fcond) + rffi.cast(lltype.Signed, op.getarg(0).getint()), + fcond) self.mc.gen_load_int(loc1.value, self.cpu.pos_exception(), fcond) self.mc.LDR_ri(loc1.value, loc1.value) @@ -397,7 +399,7 @@ if op.result in regalloc.longevity: resloc = regalloc.force_allocate_reg(op.result, args + [t, t1]) self.mc.LDR_ri(resloc.value, loc1.value) - regalloc.possibly_free_var(resloc) + regalloc.possibly_free_var(op.result) self.mc.gen_load_int(loc.value, self.cpu.pos_exception(), fcond) self.mc.MOV_ri(r.ip.value, 0) self.mc.STR_ri(r.ip.value, loc.value) @@ -406,6 +408,10 @@ regalloc.possibly_free_var(t1) return fcond + def emit_op_debug_merge_point(self, op, regalloc, fcond): + return fcond + emit_op_jit_debug = emit_op_debug_merge_point + class FieldOpAssembler(object): _mixin_ = True @@ -419,14 +425,13 @@ base_loc = regalloc.make_sure_var_in_reg(a0, imm_fine=False) value_loc = regalloc.make_sure_var_in_reg(a1, [a0], imm_fine=False) if size == 4: - f = self.mc.STR_ri + self.mc.STR_ri(value_loc.value, base_loc.value, ofs) elif size == 2: - f = self.mc.STRH_ri + self.mc.STRH_ri(value_loc.value, base_loc.value, ofs) elif size == 1: - f = self.mc.STRB_ri + self.mc.STRB_ri(value_loc.value, base_loc.value, ofs) else: assert 0 - f(value_loc.value, base_loc.value, ofs) return fcond emit_op_setfield_raw = emit_op_setfield_gc @@ -438,14 +443,13 @@ base_loc = regalloc.make_sure_var_in_reg(a0, imm_fine=False) res = regalloc.force_allocate_reg(op.result, [a0]) if size == 4: - f = self.mc.LDR_ri + self.mc.LDR_ri(res.value, base_loc.value, ofs) elif size == 2: - f = self.mc.LDRH_ri + self.mc.LDRH_ri(res.value, base_loc.value, ofs) elif size == 1: - f = self.mc.LDRB_ri + self.mc.LDRB_ri(res.value, base_loc.value, ofs) else: assert 0 - f(res.value, base_loc.value, ofs) #XXX Hack, Hack, Hack if not we_are_translated(): @@ -475,14 +479,15 @@ arraydescr = op.getdescr() assert isinstance(arraydescr, BaseArrayDescr) ofs = arraydescr.get_ofs_length(self.cpu.translate_support_code) - base_loc = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) + arg = op.getarg(0) + base_loc = regalloc.make_sure_var_in_reg(arg, imm_fine=False) regalloc.possibly_free_vars_for_op(op) - res = regalloc.force_allocate_reg(op.result, forbidden_vars=[base_loc]) + res = regalloc.force_allocate_reg(op.result, forbidden_vars=[arg]) self.mc.LDR_ri(res.value, base_loc.value, ofs) - regalloc.possibly_free_var(op.getarg(0)) - if op.result: - regalloc.possibly_free_var(op.result) + regalloc.possibly_free_var(arg) + regalloc.possibly_free_var(op.result) + return fcond def emit_op_setarrayitem_gc(self, op, regalloc, fcond): a0 = op.getarg(0) @@ -527,15 +532,14 @@ self.mc.ADD_ri(ofs_loc.value, ofs_loc.value, imm=ofs) if scale == 2: - f = self.mc.LDR_rr + self.mc.LDR_rr(res.value, base_loc.value, ofs_loc.value, cond=fcond) elif scale == 1: - f = self.mc.LDRH_rr + self.mc.LDRH_rr(res.value, base_loc.value, ofs_loc.value, cond=fcond) elif scale == 0: - f = self.mc.LDRB_rr + self.mc.LDRB_rr(res.value, base_loc.value, ofs_loc.value, cond=fcond) else: assert 0 - f(res.value, base_loc.value, ofs_loc.value, cond=fcond) #XXX Hack, Hack, Hack if not we_are_translated(): descr = op.getdescr() @@ -621,12 +625,15 @@ self.mc.STRB_ri(value_loc.value, temp.value, basesize, cond=fcond) return fcond + #from ../x86/regalloc.py:928 ff. def emit_op_copystrcontent(self, op, regalloc, fcond): self._emit_copystrcontent(op, regalloc, fcond, is_unicode=False) + return fcond def emit_op_copyunicodecontent(self, op, regalloc, fcond): self._emit_copystrcontent(op, regalloc, fcond, is_unicode=True) + return fcond def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): # compute the source address @@ -754,15 +761,15 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize/2 + + self.mc.ADD_rr(temp.value, base_loc.value, ofs_loc.value, cond=fcond, + imm=scale, shifttype=shift.LSL) if scale == 2: - f = self.mc.LDR_ri + self.mc.LDR_ri(res.value, temp.value, basesize, cond=fcond) elif scale == 1: - f = self.mc.LDRH_ri + self.mc.LDRH_ri(res.value, temp.value, basesize, cond=fcond) else: assert 0, itemsize - self.mc.ADD_rr(temp.value, base_loc.value, ofs_loc.value, cond=fcond, - imm=scale, shifttype=shift.LSL) - f(res.value, temp.value, basesize, cond=fcond) return fcond def emit_op_unicodesetitem(self, op, regalloc, fcond): @@ -779,16 +786,16 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize/2 + + self.mc.ADD_rr(temp.value, base_loc.value, ofs_loc.value, cond=fcond, + imm=scale, shifttype=shift.LSL) if scale == 2: - f = self.mc.STR_ri + self.mc.STR_ri(value_loc.value, temp.value, basesize, cond=fcond) elif scale == 1: - f = self.mc.STRH_ri + self.mc.STRH_ri(value_loc.value, temp.value, basesize, cond=fcond) else: assert 0, itemsize - self.mc.ADD_rr(temp.value, base_loc.value, ofs_loc.value, cond=fcond, - imm=scale, shifttype=shift.LSL) - f(value_loc.value, temp.value, basesize, cond=fcond) return fcond class ForceOpAssembler(object): @@ -831,7 +838,7 @@ # check value t = TempBox() resloc = regalloc.force_allocate_reg(resbox) - loc = regalloc.force_allocate_reg(t, [r.r0]) + loc = regalloc.force_allocate_reg(t) self.mc.gen_load_int(loc.value, value) self.mc.CMP_rr(resloc.value, loc.value) regalloc.possibly_free_var(resbox) @@ -964,14 +971,13 @@ size = scale * 2 ofs = ofs_length if size == 4: - f = self.mc.STR_ri + self.mc.STR_ri(value_loc.value, base_loc.value, ofs) elif size == 2: - f = self.mc.STRH_ri + self.mc.STRH_ri(value_loc.value, base_loc.value, ofs) elif size == 1: - f = self.mc.STRB_ri + self.mc.STRB_ri(value_loc.value, base_loc.value, ofs) else: assert 0 - f(value_loc.value, base_loc.value, ofs) def emit_op_new(self, op, regalloc, fcond): arglocs = self._prepare_args_for_new_op(op.getdescr(), regalloc) @@ -994,7 +1000,7 @@ loc = regalloc.loc(result) if self.cpu.vtable_offset is not None: assert loc.is_reg() - adr = self.cpu.cast_adr_to_int(vtable.getint()) + adr = rffi.cast(lltype.Signed, vtable.getint()) t = TempBox() loc_vtable = regalloc.force_allocate_reg(t) self.mc.gen_load_int(loc_vtable.value, adr) @@ -1016,8 +1022,9 @@ # boehm GC (XXX kill the following code at some point) itemsize, scale, basesize, ofs_length, _ = ( self._unpack_arraydescr(op.getdescr())) - return self._malloc_varsize(basesize, ofs_length, scale, + self._malloc_varsize(basesize, ofs_length, scale, op.getarg(0), op.result, regalloc) + return fcond def emit_op_newstr(self, op, regalloc, fcond): @@ -1031,8 +1038,9 @@ ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - return self._malloc_varsize(ofs_items, ofs, 1, op.getarg(0), + self._malloc_varsize(ofs_items, ofs, 1, op.getarg(0), op.result, regalloc) + return fcond @@ -1047,8 +1055,9 @@ ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = self._get_unicode_item_scale() - return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0), + self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0), op.result, regalloc) + return fcond def _get_unicode_item_scale(self): _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Wed Dec 1 18:33:12 2010 @@ -2,7 +2,7 @@ RegisterManager, compute_vars_longevity from pypy.jit.backend.arm import registers as r from pypy.jit.backend.arm import locations -from pypy.jit.metainterp.history import ConstInt +from pypy.jit.metainterp.history import ConstInt, ConstPtr from pypy.rpython.lltypesystem import rffi, lltype class ARMRegisterManager(RegisterManager): @@ -18,6 +18,7 @@ if isinstance(c, ConstInt): return locations.ImmLocation(c.value) else: + assert isinstance(c, ConstPtr) return locations.ImmLocation(rffi.cast(lltype.Signed, c.value)) def call_result_location(self, v): Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/runner.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/runner.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/runner.py Wed Dec 1 18:33:12 2010 @@ -17,10 +17,10 @@ translate_support_code, gcdescr) self.assembler = AssemblerARM(self) - def compile_loop(self, inputargs, operations, looptoken): + def compile_loop(self, inputargs, operations, looptoken, log=True): self.assembler.assemble_loop(inputargs, operations, looptoken) - def compile_bridge(self, faildescr, inputargs, operations): + def compile_bridge(self, faildescr, inputargs, operations, log=True): self.assembler.assemble_bridge(faildescr, inputargs, operations) def set_future_value_int(self, index, intvalue): @@ -39,8 +39,10 @@ return self.assembler.fail_boxes_count def clear_latest_values(self, count): - # XXX TODO - pass + setitem = self.assembler.fail_boxes_ptr.setitem + null = lltype.nullptr(llmemory.GCREF.TO) + for index in range(count): + setitem(index, null) def execute_token(self, executable_token): #i = [self.get_latest_value_int(x) for x in range(10)] @@ -52,18 +54,22 @@ return self.get_fail_descr_from_number(fail_index) def _execute_call(self, func): - prev_interpreter = LLInterpreter.current_interpreter - LLInterpreter.current_interpreter = self.debug_ll_interpreter + prev_interpreter = None + if not self.translate_support_code: + prev_interpreter = LLInterpreter.current_interpreter + LLInterpreter.current_interpreter = self.debug_ll_interpreter res = 0 try: res = func() finally: - LLInterpreter.current_interpreter = prev_interpreter + if not self.translate_support_code: + LLInterpreter.current_interpreter = prev_interpreter return res - def cast_ptr_to_int(self, x): + @staticmethod + def cast_ptr_to_int(x): adr = llmemory.cast_ptr_to_adr(x) - return self.cast_adr_to_int(adr) + return ArmCPU.cast_adr_to_int(adr) def force(self, addr_of_force_index): TP = rffi.CArrayPtr(lltype.Signed) Modified: pypy/branch/arm-backend/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/llsupport/llmodel.py Wed Dec 1 18:33:12 2010 @@ -34,7 +34,7 @@ self.rtyper = rtyper self.stats = stats self.translate_support_code = translate_support_code - if translate_support_code: + if translate_support_code and rtyper is not None: translator = rtyper.annotator.translator else: translator = None Modified: pypy/branch/arm-backend/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/model.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/model.py Wed Dec 1 18:33:12 2010 @@ -8,6 +8,10 @@ done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + # for heaptracker + _all_size_descrs_with_vtable = None + _vtable_to_descr_dict = None + def __init__(self): self.fail_descr_list = [] @@ -44,7 +48,7 @@ """Assemble the bridge. The FailDescr is the descr of the original guard that failed. """ - raise NotImplementedError + raise NotImplementedError def execute_token(self, looptoken): """Execute the generated code referenced by the looptoken. From arigo at codespeak.net Wed Dec 1 18:36:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 1 Dec 2010 18:36:25 +0100 (CET) Subject: [pypy-svn] r79722 - pypy/trunk/pypy/jit/backend/x86/test Message-ID: <20101201173625.54C04282BEF@codespeak.net> Author: arigo Date: Wed Dec 1 18:36:23 2010 New Revision: 79722 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Log: Kill test that no longer makes sense. Modified: pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_regalloc.py Wed Dec 1 18:36:23 2010 @@ -9,7 +9,7 @@ from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ - FloatConstants, is_comparison_or_ovf_op + is_comparison_or_ovf_op from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -520,16 +520,6 @@ self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] - def test_float_overflow_const_list(self): - ops = ['[f0]'] - BASE_CONSTANT_SIZE = FloatConstants.BASE_CONSTANT_SIZE - for i in range(BASE_CONSTANT_SIZE * 2): - ops.append('f%d = float_add(f%d, 3.5)' % (i + 1, i)) - ops.append('finish(f%d)' % (BASE_CONSTANT_SIZE * 2)) - ops = "\n".join(ops) - self.interpret(ops, [0.1]) - assert abs(self.getfloat(0) - (BASE_CONSTANT_SIZE * 2) * 3.5 - 0.1) < 0.00001 - def test_lt_const(self): ops = ''' [f0] From arigo at codespeak.net Wed Dec 1 18:36:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 1 Dec 2010 18:36:45 +0100 (CET) Subject: [pypy-svn] r79723 - in pypy/trunk/pypy/jit/backend: llgraph test Message-ID: <20101201173645.B7E21282BF0@codespeak.net> Author: arigo Date: Wed Dec 1 18:36:43 2010 New Revision: 79723 Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py pypy/trunk/pypy/jit/backend/test/runner_test.py Log: Add a test for DEBUG_MERGE_POINT and JIT_DEBUG. Modified: pypy/trunk/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/trunk/pypy/jit/backend/llgraph/llimpl.py Wed Dec 1 18:36:43 2010 @@ -583,7 +583,12 @@ def op_debug_merge_point(self, _, value, recdepth): from pypy.jit.metainterp.warmspot import get_stats loc = ConstPtr(value)._get_str() - get_stats().add_merge_point_location(loc) + try: + stats = get_stats() + except AttributeError: + pass + else: + stats.add_merge_point_location(loc) def op_guard_true(self, _, value): if not value: Modified: pypy/trunk/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/trunk/pypy/jit/backend/test/runner_test.py (original) +++ pypy/trunk/pypy/jit/backend/test/runner_test.py Wed Dec 1 18:36:43 2010 @@ -1203,6 +1203,13 @@ yield nan_and_infinity, rop.FLOAT_GT, operator.gt, all_cases_binary yield nan_and_infinity, rop.FLOAT_GE, operator.ge, all_cases_binary + def test_noops(self): + c_box = self.alloc_string("hi there").constbox() + c_nest = ConstInt(0) + self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest], 'void') + self.execute_operation(rop.JIT_DEBUG, [c_box, c_nest, c_nest, + c_nest, c_nest], 'void') + class LLtypeBackendTest(BaseBackendTest): From afa at codespeak.net Wed Dec 1 20:08:49 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 1 Dec 2010 20:08:49 +0100 (CET) Subject: [pypy-svn] r79724 - in pypy/branch/fast-forward/pypy: module/__builtin__ objspace/std Message-ID: <20101201190849.B7722282BEC@codespeak.net> Author: afa Date: Wed Dec 1 20:08:45 2010 New Revision: 79724 Modified: pypy/branch/fast-forward/pypy/module/__builtin__/app_operation.py pypy/branch/fast-forward/pypy/objspace/std/complexobject.py pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Log: Some fixes to translate with cpython2.5 (or pypy-1.4!) - use the rarithmetic.copysign(), which has a fall-back implementation - don't use the global builtin "format()", it confuses geninterp. Modified: pypy/branch/fast-forward/pypy/module/__builtin__/app_operation.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__builtin__/app_operation.py (original) +++ pypy/branch/fast-forward/pypy/module/__builtin__/app_operation.py Wed Dec 1 20:08:45 2010 @@ -1,4 +1,4 @@ def bin(x): if not isinstance(x, (int, long)): raise TypeError("must be int or long") - return format(x, "#b") + return x.__format__("#b") Modified: pypy/branch/fast-forward/pypy/objspace/std/complexobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/complexobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/complexobject.py Wed Dec 1 20:08:45 2010 @@ -3,7 +3,7 @@ from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all from pypy.objspace.std.floatobject import W_FloatObject, _hash_float -from pypy.rlib.rarithmetic import formatd, isinf, isnan +from pypy.rlib.rarithmetic import formatd, isinf, isnan, copysign import math @@ -273,16 +273,16 @@ return format_float(x, "%.12g") def repr__Complex(space, w_complex): - if w_complex.realval == 0 and math.copysign(1., w_complex.realval) == 1.: + if w_complex.realval == 0 and copysign(1., w_complex.realval) == 1.: return space.wrap(repr_format(w_complex.imagval) + 'j') - sign = (math.copysign(1., w_complex.imagval) == 1.) and '+' or '' + sign = (copysign(1., w_complex.imagval) == 1.) and '+' or '' return space.wrap('(' + repr_format(w_complex.realval) + sign + repr_format(w_complex.imagval) + 'j)') def str__Complex(space, w_complex): - if w_complex.realval == 0 and math.copysign(1., w_complex.realval) == 1.: + if w_complex.realval == 0 and copysign(1., w_complex.realval) == 1.: return space.wrap(str_format(w_complex.imagval) + 'j') - sign = (math.copysign(1., w_complex.imagval) == 1.) and '+' or '' + sign = (copysign(1., w_complex.imagval) == 1.) and '+' or '' return space.wrap('(' + str_format(w_complex.realval) + sign + str_format(w_complex.imagval) + 'j)') Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Wed Dec 1 20:08:45 2010 @@ -9,7 +9,7 @@ from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan -from pypy.rlib.rarithmetic import formatd, LONG_BIT, INFINITY +from pypy.rlib.rarithmetic import formatd, LONG_BIT, INFINITY, copysign from pypy.rlib.rbigint import rbigint from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rfloat @@ -102,7 +102,7 @@ if isinf(value) or isnan(value): return str__Float(space, w_float) if value == 0.0: - if math.copysign(1., value) == -1.: + if copysign(1., value) == -1.: return space.wrap("-0x0.0p+0") else: return space.wrap("0x0.0p+0") From hakanardo at codespeak.net Wed Dec 1 20:10:52 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 1 Dec 2010 20:10:52 +0100 (CET) Subject: [pypy-svn] r79725 - in pypy/branch/jit-unroll-loops/pypy/jit/metainterp: optimizeopt test Message-ID: <20101201191052.0CD1A282BF0@codespeak.net> Author: hakanardo Date: Wed Dec 1 20:10:51 2010 New Revision: 79725 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py Log: more rigorous way of matchin ops between preamble and loop Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py Wed Dec 1 20:10:51 2010 @@ -6,7 +6,7 @@ from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp.optimizeutil import InvalidLoop, RetraceLoop -# FXIME: Introduce some VirtualOptimizer super class instead +# FIXME: Introduce some VirtualOptimizer super class instead def optimize_unroll(metainterp_sd, loop, optimizations): opt = UnrollOptimizer(metainterp_sd, loop, optimizations) @@ -20,17 +20,18 @@ self.argmap[inputargs[i]] = jump_args[i] self.snapshot_map = {None: None} - def inline_op(self, op): - newop = op #.clone() FIXME: move clone here + def inline_op(self, newop, ignore_result=False, clone=True): + if clone: + newop = newop.clone() args = newop.getarglist() newop.initarglist([self.inline_arg(a) for a in args]) - if op.is_guard(): + if newop.is_guard(): args = newop.getfailargs() if args: newop.setfailargs([self.inline_arg(a) for a in args]) - if newop.result: + if newop.result and not ignore_result: old_result = newop.result newop.result = newop.result.clonebox() self.argmap[old_result] = newop.result @@ -63,11 +64,9 @@ def __init__(self, metainterp_sd, loop, optimizations): self.optimizer = Optimizer(metainterp_sd, loop, optimizations) self.cloned_operations = [] - self.originalop = {} for op in self.optimizer.loop.operations: newop = op.clone() self.cloned_operations.append(newop) - self.originalop[newop] = op def propagate_all_forward(self): loop = self.optimizer.loop @@ -169,11 +168,12 @@ # This loop is equivalent to the main optimization loop in # Optimizer.propagate_all_forward for newop in loop_operations: - #print 'N:', newop + print 'N:', newop if newop.getopnum() == rop.JUMP: newop.initarglist(inputargs) - newop = inliner.inline_op(newop) - + newop = inliner.inline_op(newop, clone=False) + print 'I:', newop + self.optimizer.first_optimization.propagate_forward(newop) # Remove jump to make sure forced code are placed before it @@ -206,17 +206,31 @@ self.optimizer.newoperations.append(jmp) return inputargs - def sameop(self, preambleop, loopop): - #if preambleop.getopnum() != loopop.getopnum(): - # return False - #pargs = preambleop.getarglist() - #largs = loopop.getarglist() - #if len(pargs) != len(largs): - # return False - try: - return self.originalop[loopop] is preambleop - except KeyError: + def sameop(self, op1, op2): + if op1.getopnum() != op2.getopnum(): + return False + + args1 = op1.getarglist() + args2 = op2.getarglist() + if len(args1) != len(args2): return False + for i in range(len(args1)): + box1, box2 = args1[i], args2[i] + if box1 in self.optimizer.values: + box1 = self.optimizer.values[box1].force_box() + if box2 in self.optimizer.values: + box2 = self.optimizer.values[box2].force_box() + + if box1 is not box2: + return False + + if not op1.is_guard(): + descr1 = op1.getdescr() + descr2 = op2.getdescr() + if descr1 is not descr2: + return False + + return True def create_short_preamble(self, preamble, loop): #return None # Dissable @@ -228,8 +242,16 @@ short_preamble = [] loop_i = preamble_i = 0 while preamble_i < len(preamble_ops)-1: + op = preamble_ops[preamble_i] - if self.sameop(op, loop_ops[loop_i]) \ + try: + newop = self.inliner.inline_op(op, True) + except KeyError: + debug_print("create_short_preamble failed due to", + "new boxes created during optimization") + return None + + if self.sameop(newop, loop_ops[loop_i]) \ and loop_i < len(loop_ops)-1: loop_i += 1 else: @@ -272,6 +294,10 @@ jmp.setdescr(loop.token) short_preamble.append(jmp) + print + print preamble.inputargs + print '\n'.join([str(o) for o in short_preamble]) + # Check that boxes used as arguemts are produced. seen = {} for box in preamble.inputargs: @@ -281,6 +307,7 @@ if isinstance(box, Const): continue if box not in seen: + print box debug_print("create_short_preamble failed due to", "op arguments not produced") return None @@ -366,7 +393,7 @@ inliner = Inliner(loop_args, jump_args) for op in loop_operations: - newop = inliner.inline_op(op.clone()) + newop = inliner.inline_op(op) if not dryrun: self.emit_operation(newop) Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py Wed Dec 1 20:10:51 2010 @@ -1458,8 +1458,8 @@ return x res = self.meta_interp(f, [299], listops=True) assert res == f(299) - self.check_loops(guard_class=0, guard_value=1) - self.check_loops(guard_class=0, guard_value=4, everywhere=True) + self.check_loops(guard_class=0, guard_value=3) + self.check_loops(guard_class=0, guard_value=6, everywhere=True) def test_merge_guardnonnull_guardclass(self): from pypy.rlib.objectmodel import instantiate @@ -1488,9 +1488,9 @@ res = self.meta_interp(f, [299], listops=True) assert res == f(299) self.check_loops(guard_class=0, guard_nonnull=0, - guard_nonnull_class=1, guard_isnull=0) + guard_nonnull_class=2, guard_isnull=1) self.check_loops(guard_class=0, guard_nonnull=0, - guard_nonnull_class=3, guard_isnull=1, + guard_nonnull_class=4, guard_isnull=2, everywhere=True) def test_merge_guardnonnull_guardvalue(self): @@ -1518,10 +1518,10 @@ return x res = self.meta_interp(f, [299], listops=True) assert res == f(299) - self.check_loops(guard_class=0, guard_nonnull=0, guard_value=1, - guard_nonnull_class=0, guard_isnull=0) - self.check_loops(guard_class=0, guard_nonnull=0, guard_value=3, - guard_nonnull_class=0, guard_isnull=1, + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=2, + guard_nonnull_class=0, guard_isnull=1) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=4, + guard_nonnull_class=0, guard_isnull=2, everywhere=True) def test_merge_guardnonnull_guardvalue_2(self): @@ -1549,10 +1549,10 @@ return x res = self.meta_interp(f, [299], listops=True) assert res == f(299) - self.check_loops(guard_class=0, guard_nonnull=0, guard_value=1, - guard_nonnull_class=0, guard_isnull=0) - self.check_loops(guard_class=0, guard_nonnull=0, guard_value=3, - guard_nonnull_class=0, guard_isnull=1, + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=2, + guard_nonnull_class=0, guard_isnull=1) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=4, + guard_nonnull_class=0, guard_isnull=2, everywhere=True) def test_merge_guardnonnull_guardclass_guardvalue(self): @@ -1583,10 +1583,10 @@ return x res = self.meta_interp(f, [399], listops=True) assert res == f(399) - self.check_loops(guard_class=0, guard_nonnull=0, guard_value=1, - guard_nonnull_class=0, guard_isnull=0) - self.check_loops(guard_class=0, guard_nonnull=0, guard_value=4, - guard_nonnull_class=0, guard_isnull=1, + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=3, + guard_nonnull_class=0, guard_isnull=1) + self.check_loops(guard_class=0, guard_nonnull=0, guard_value=6, + guard_nonnull_class=0, guard_isnull=2, everywhere=True) def test_residual_call_doesnt_lose_info(self): From hakanardo at codespeak.net Wed Dec 1 20:32:08 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 1 Dec 2010 20:32:08 +0100 (CET) Subject: [pypy-svn] r79726 - pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt Message-ID: <20101201193208.75A9D282BD4@codespeak.net> Author: hakanardo Date: Wed Dec 1 20:32:07 2010 New Revision: 79726 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py Log: cleanups Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py Wed Dec 1 20:32:07 2010 @@ -114,40 +114,51 @@ else: loop.preamble.token.short_preamble = [short_loop] + # Clone ops and boxes to get private versions and + newargs = [a.clonebox() for a in short_loop.inputargs] + inliner = Inliner(short_loop.inputargs, newargs) + short_loop.inputargs = newargs + ops = [inliner.inline_op(op) for op in short_loop.operations] + short_loop.operations = ops + + # Forget the values to allow them to be freed + for box in short_loop.inputargs: + box.forget_value() + for op in short_loop.operations: + if op.result: + op.result.forget_value() - - # Clone ops and boxes to get private versions and forget the - # values to allow them to be freed - boxmap = {} - for i in range(len(short_loop.inputargs)): - box = short_loop.inputargs[i] - newbox = box.clonebox() - boxmap[box] = newbox - newbox.forget_value() - short_loop.inputargs[i] = newbox - for i in range(len(short)): - oldop = short[i] - op = oldop.clone() - args = [] - for a in op.getarglist(): - if not isinstance(a, Const): - a = boxmap[a] - args.append(a) - op.initarglist(args) - if op.is_guard(): + if False: + boxmap = {} + for i in range(len(short_loop.inputargs)): + box = short_loop.inputargs[i] + newbox = box.clonebox() + boxmap[box] = newbox + newbox.forget_value() + short_loop.inputargs[i] = newbox + for i in range(len(short)): + oldop = short[i] + op = oldop.clone() args = [] - for a in op.getfailargs(): + for a in op.getarglist(): if not isinstance(a, Const): a = boxmap[a] args.append(a) - op.setfailargs(args) - box = op.result - if box: - newbox = box.clonebox() - boxmap[box] = newbox - newbox.forget_value() - op.result = newbox - short[i] = op + op.initarglist(args) + if op.is_guard(): + args = [] + for a in op.getfailargs(): + if not isinstance(a, Const): + a = boxmap[a] + args.append(a) + op.setfailargs(args) + box = op.result + if box: + newbox = box.clonebox() + boxmap[box] = newbox + newbox.forget_value() + op.result = newbox + short[i] = op def inline(self, loop_operations, loop_args, jump_args): @@ -168,11 +179,9 @@ # This loop is equivalent to the main optimization loop in # Optimizer.propagate_all_forward for newop in loop_operations: - print 'N:', newop if newop.getopnum() == rop.JUMP: newop.initarglist(inputargs) newop = inliner.inline_op(newop, clone=False) - print 'I:', newop self.optimizer.first_optimization.propagate_forward(newop) @@ -187,7 +196,6 @@ # FIXME: Should also loop over operations added by forcing things in this loop for op in newoperations: - #print 'E: ', str(op) boxes_created_this_iteration[op.result] = True args = op.getarglist() if op.is_guard(): @@ -294,10 +302,6 @@ jmp.setdescr(loop.token) short_preamble.append(jmp) - print - print preamble.inputargs - print '\n'.join([str(o) for o in short_preamble]) - # Check that boxes used as arguemts are produced. seen = {} for box in preamble.inputargs: @@ -307,7 +311,6 @@ if isinstance(box, Const): continue if box not in seen: - print box debug_print("create_short_preamble failed due to", "op arguments not produced") return None From hakanardo at codespeak.net Wed Dec 1 21:46:36 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Wed, 1 Dec 2010 21:46:36 +0100 (CET) Subject: [pypy-svn] r79727 - pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt Message-ID: <20101201204636.B6F91282B9D@codespeak.net> Author: hakanardo Date: Wed Dec 1 21:46:33 2010 New Revision: 79727 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py Log: Replacements mode that replaces the long preamble with the short now working again Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py Wed Dec 1 21:46:33 2010 @@ -101,12 +101,20 @@ if False: # FIXME: This should save some memory but requires # a lot of tests to be fixed... - loop.preamble.operations = short - short_loop = loop.preamble - else: - short_loop = TreeLoop('short preamble') - short_loop.inputargs = loop.preamble.inputargs[:] - short_loop.operations = short + loop.preamble.operations = short[:] + + # Turn guards into conditional jumps to the preamble + for i in range(len(short)): + op = short[i] + if op.is_guard(): + op = op.clone() + op.setfailargs(loop.preamble.inputargs) + op.setjumptarget(loop.preamble.token) + short[i] = op + + short_loop = TreeLoop('short preamble') + short_loop.inputargs = loop.preamble.inputargs[:] + short_loop.operations = short assert isinstance(loop.preamble.token, LoopToken) if loop.preamble.token.short_preamble: @@ -269,6 +277,7 @@ "at position: ", preamble_i) return None short_preamble.append(op) + state.update(op) preamble_i += 1 @@ -317,15 +326,6 @@ if op.result: seen[op.result] = True - # Turn guards into conditional jumps to the preamble - for i in range(len(short_preamble)): - op = short_preamble[i] - if op.is_guard(): - op = op.clone() - op.setfailargs(preamble.inputargs) - op.setjumptarget(preamble.token) - short_preamble[i] = op - return short_preamble class ExeState(object): From afa at codespeak.net Thu Dec 2 00:45:41 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 2 Dec 2010 00:45:41 +0100 (CET) Subject: [pypy-svn] r79728 - in pypy/trunk: lib_pypy/_ctypes pypy/module/test_lib_pypy/ctypes_tests Message-ID: <20101201234541.6D2D9282B90@codespeak.net> Author: afa Date: Thu Dec 2 00:45:35 2010 New Revision: 79728 Modified: pypy/trunk/lib_pypy/_ctypes/array.py pypy/trunk/lib_pypy/_ctypes/basics.py pypy/trunk/lib_pypy/_ctypes/builtin.py pypy/trunk/lib_pypy/_ctypes/function.py pypy/trunk/lib_pypy/_ctypes/pointer.py pypy/trunk/lib_pypy/_ctypes/primitive.py pypy/trunk/lib_pypy/_ctypes/structure.py pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py Log: Merge the most important _ctypes fixes from the fast-forward branch - better keepalives should avoid some crashes - sub-classes of Struct should put the base fields first - respect the "from_param" conversion method ... 10 less failures in the (skipped!) cpython test suite Modified: pypy/trunk/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/array.py (original) +++ pypy/trunk/lib_pypy/_ctypes/array.py Thu Dec 2 00:45:35 2010 @@ -99,6 +99,9 @@ if len(value) > self._length_: raise ValueError("Invalid length") value = self(*value) + elif not isinstance(value, self): + raise TypeError("expected string or Unicode object, %s found" + % (value.__class__.__name__,)) else: if isinstance(value, tuple): if len(value) > self._length_: @@ -107,22 +110,43 @@ return _CDataMeta.from_param(self, value) def array_get_slice_params(self, index): - if index.step is not None: - raise TypeError("3 arg slices not supported (for no reason)") - start = index.start or 0 - stop = index.stop or self._length_ - return start, stop + if hasattr(self, '_length_'): + start, stop, step = index.indices(self._length_) + else: + step = index.step + if step is None: + step = 1 + start = index.start + stop = index.stop + if start is None: + if step > 0: + start = 0 + else: + raise ValueError("slice start is required for step < 0") + if stop is None: + raise ValueError("slice stop is required") + + return start, stop, step def array_slice_setitem(self, index, value): - start, stop = self._get_slice_params(index) - if stop - start != len(value): + start, stop, step = self._get_slice_params(index) + + if ((step < 0 and stop >= start) or + (step > 0 and start >= stop)): + slicelength = 0 + elif step < 0: + slicelength = (stop - start + 1) / step + 1 + else: + slicelength = (stop - start - 1) / step + 1; + + if slicelength != len(value): raise ValueError("Can only assign slices of the same length") - for i in range(start, stop): - self[i] = value[i - start] + for i, j in enumerate(range(start, stop, step)): + self[j] = value[i] def array_slice_getitem(self, index): - start, stop = self._get_slice_params(index) - l = [self[i] for i in range(start, stop)] + start, stop, step = self._get_slice_params(index) + l = [self[i] for i in range(start, stop, step)] letter = getattr(self._type_, '_type_', None) if letter == 'c': return "".join(l) @@ -134,8 +158,12 @@ __metaclass__ = ArrayMeta _ffiargshape = 'P' - def __init__(self, *args): + def __new__(cls, *args): + self = _CData.__new__(cls, *args) self._buffer = self._ffiarray(self._length_, autofree=True) + return self + + def __init__(self, *args): for i, arg in enumerate(args): self[i] = arg @@ -162,9 +190,10 @@ self._slice_setitem(index, value) return index = self._fix_index(index) - if ensure_objects(value) is not None: - store_reference(self, index, value._objects) - arg = self._type_._CData_value(value) + cobj = self._type_.from_param(value) + if ensure_objects(cobj) is not None: + store_reference(self, index, cobj._objects) + arg = cobj._get_buffer_value() if self._type_._fficompositesize is None: self._buffer[index] = arg # something more sophisticated, cannot set field directly @@ -183,7 +212,7 @@ return self._length_ def _get_buffer_for_param(self): - return CArgObject(self._buffer.byptr()) + return CArgObject(self, self._buffer.byptr()) def _get_buffer_value(self): return self._buffer.buffer Modified: pypy/trunk/lib_pypy/_ctypes/basics.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/basics.py (original) +++ pypy/trunk/lib_pypy/_ctypes/basics.py Thu Dec 2 00:45:35 2010 @@ -46,21 +46,6 @@ else: return self.from_param(as_parameter) - def _CData_input(self, value): - """Used when data enters into ctypes from user code. 'value' is - some user-specified Python object, which is converted into a _rawffi - array of length 1 containing the same value according to the - type 'self'. - """ - cobj = self.from_param(value) - return cobj, cobj._get_buffer_for_param() - - def _CData_value(self, value): - cobj = self.from_param(value) - # we don't care here if this stuff will live afterwards, as we're - # interested only in value anyway - return cobj._get_buffer_value() - def _CData_output(self, resbuffer, base=None, index=-1): #assert isinstance(resbuffer, _rawffi.ArrayInstance) """Used when data exits ctypes and goes into user code. @@ -92,13 +77,20 @@ """ simple wrapper around buffer, just for the case of freeing it afterwards """ - def __init__(self, buffer): + def __init__(self, obj, buffer): + self._obj = obj self._buffer = buffer def __del__(self): self._buffer.free() self._buffer = None + def __repr__(self): + return repr(self._obj) + + def __eq__(self, other): + return self._obj == other + class _CData(object): """ The most basic object for all ctypes types """ @@ -128,7 +120,7 @@ return buffer(self._buffer) def _get_b_base(self): - return self._objects + return self._base _b_base_ = property(_get_b_base) _b_needsfree_ = False Modified: pypy/trunk/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/builtin.py (original) +++ pypy/trunk/lib_pypy/_ctypes/builtin.py Thu Dec 2 00:45:35 2010 @@ -11,7 +11,8 @@ def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_void_p._CData_value(addr) + cobj = ctypes.c_void_p.from_param(addr) + arg = cobj._get_buffer_value() return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -22,7 +23,8 @@ def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_void_p._CData_value(addr) + cobj = ctypes.c_void_p.from_param(addr) + arg = cobj._get_buffer_value() # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/trunk/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/function.py (original) +++ pypy/trunk/lib_pypy/_ctypes/function.py Thu Dec 2 00:45:35 2010 @@ -35,6 +35,7 @@ _argtypes_ = None _restype_ = None + _errcheck_ = None _flags_ = 0 _ffiargshape = 'P' _ffishape = 'P' @@ -53,7 +54,15 @@ return self._argtypes_ def _setargtypes(self, argtypes): self._ptr = None - self._argtypes_ = argtypes + if argtypes is None: + self._argtypes_ = None + else: + for i, argtype in enumerate(argtypes): + if not hasattr(argtype, 'from_param'): + raise TypeError( + "item %d in _argtypes_ has no from_param method" % ( + i + 1,)) + self._argtypes_ = argtypes argtypes = property(_getargtypes, _setargtypes) def _getrestype(self): @@ -72,9 +81,24 @@ del self._restype_ restype = property(_getrestype, _setrestype, _delrestype) + def _geterrcheck(self): + return getattr(self, '_errcheck_', None) + def _seterrcheck(self, errcheck): + if not callable(errcheck): + raise TypeError("The errcheck attribute must be callable") + self._errcheck_ = errcheck + def _delerrcheck(self): + try: + del self._errcheck_ + except AttributeError: + pass + errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck) + def _ffishapes(self, args, restype): argtypes = [arg._ffiargshape for arg in args] if restype is not None: + if not isinstance(restype, _CDataMeta): + raise TypeError("invalid result type for callback function") restype = restype._ffiargshape else: restype = 'O' # void @@ -140,6 +164,7 @@ def __call__(self, *args): if self.callable is not None: + args = args[:len(self._argtypes_)] try: res = self.callable(*args) except: @@ -162,13 +187,29 @@ thisarg = None if argtypes is None: - argtypes = self._guess_argtypes(args) - argtypes, argsandobjs = self._wrap_args(argtypes, args) + argtypes = [] + args = self._convert_args(argtypes, args) + argtypes = [type(arg) for arg in args] restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype, thisarg) - resbuffer = funcptr(*[arg._buffer for _, arg in argsandobjs]) - return self._build_result(restype, resbuffer, argtypes, argsandobjs) + resbuffer = funcptr(*[arg._get_buffer_for_param()._buffer + for arg in args]) + result = self._build_result(restype, resbuffer, argtypes, args) + + # The 'errcheck' protocol + if self._errcheck_: + v = self._errcheck_(result, self, args) + # If the errcheck funtion failed, let it throw + # If the errcheck function returned callargs unchanged, + # continue normal processing. + # If the errcheck function returned something else, + # use that as result. + if v is not args: + result = v + + return result + def _getfuncptr(self, argtypes, restype, thisarg=None): if self._ptr is not None and argtypes is self._argtypes_: @@ -212,31 +253,33 @@ raise @staticmethod - def _guess_argtypes(args): + def _conv_param(argtype, arg, index): from ctypes import c_char_p, c_wchar_p, c_void_p, c_int - res = [] - for arg in args: - if hasattr(arg, '_as_parameter_'): - arg = arg._as_parameter_ - if isinstance(arg, str): - res.append(c_char_p) - elif isinstance(arg, unicode): - res.append(c_wchar_p) - elif isinstance(arg, _CData): - res.append(type(arg)) - elif arg is None: - res.append(c_void_p) - #elif arg == 0: - # res.append(c_void_p) - elif isinstance(arg, (int, long)): - res.append(c_int) - else: - raise TypeError("Don't know how to handle %s" % (arg,)) - return res + if argtype is not None: + arg = argtype.from_param(arg) + if hasattr(arg, '_as_parameter_'): + arg = arg._as_parameter_ + + if isinstance(arg, _CData): + # The usual case when argtype is defined + cobj = arg + elif isinstance(arg, str): + cobj = c_char_p(arg) + elif isinstance(arg, unicode): + cobj = c_wchar_p(arg) + elif arg is None: + cobj = c_void_p() + elif isinstance(arg, (int, long)): + cobj = c_int(arg) + else: + raise TypeError("Don't know how to handle %s" % (arg,)) - def _wrap_args(self, argtypes, args): + return cobj + + def _convert_args(self, argtypes, args): wrapped_args = [] consumed = 0 + for i, argtype in enumerate(argtypes): defaultvalue = None if i > 0 and self._paramflags is not None: @@ -263,7 +306,7 @@ val = defaultvalue if val is None: val = 0 - wrapped = argtype._CData_input(val) + wrapped = self._conv_param(argtype, val, consumed) wrapped_args.append(wrapped) continue else: @@ -278,23 +321,22 @@ raise TypeError("Not enough arguments") try: - wrapped = argtype._CData_input(arg) - except (UnicodeError, TypeError), e: + wrapped = self._conv_param(argtype, arg, consumed) + except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) wrapped_args.append(wrapped) consumed += 1 if len(wrapped_args) < len(args): extra = args[len(wrapped_args):] - extra_types = self._guess_argtypes(extra) - for arg, argtype in zip(extra, extra_types): + argtypes = list(argtypes) + for i, arg in enumerate(extra): try: - wrapped = argtype._CData_input(arg) - except (UnicodeError, TypeError), e: + wrapped = self._conv_param(None, arg, i) + except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) wrapped_args.append(wrapped) - argtypes = list(argtypes) + extra_types - return argtypes, wrapped_args + return wrapped_args def _build_result(self, restype, resbuffer, argtypes, argsandobjs): """Build the function result: @@ -307,7 +349,7 @@ if self._com_index: if resbuffer[0] & 0x80000000: raise get_com_error(resbuffer[0], - self._com_iid, argsandobjs[0][0]) + self._com_iid, argsandobjs[0]) else: retval = int(resbuffer[0]) elif restype is not None: @@ -326,8 +368,8 @@ results = [] if self._paramflags: - for argtype, (obj, _), paramflag in zip(argtypes[1:], argsandobjs[1:], - self._paramflags): + for argtype, obj, paramflag in zip(argtypes[1:], argsandobjs[1:], + self._paramflags): if len(paramflag) == 2: idlflag, name = paramflag elif len(paramflag) == 3: Modified: pypy/trunk/lib_pypy/_ctypes/pointer.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/pointer.py (original) +++ pypy/trunk/lib_pypy/_ctypes/pointer.py Thu Dec 2 00:45:35 2010 @@ -1,7 +1,8 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, cdata_from_address -from _ctypes.basics import sizeof, byref, keepalive_key +from _ctypes.basics import keepalive_key, store_reference, ensure_objects +from _ctypes.basics import sizeof, byref from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem @@ -99,7 +100,10 @@ return self._type_._CData_output(self._subarray(index), self, index) def __setitem__(self, index, value): - self._subarray(index)[0] = self._type_._CData_value(value) + cobj = self._type_.from_param(value) + if ensure_objects(cobj) is not None: + store_reference(self, index, cobj._objects) + self._subarray(index)[0] = cobj._get_buffer_value() def __nonzero__(self): return self._buffer[0] != 0 @@ -110,29 +114,32 @@ if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): raise TypeError("cast() argument 2 must be a pointer type, not %s" % (tp,)) - if isinstance(obj, Array): - ptr = tp.__new__(tp) - ptr._buffer = tp._ffiarray(1, autofree=True) - ptr._buffer[0] = obj._buffer - return ptr if isinstance(obj, (int, long)): result = tp() result._buffer[0] = obj return result - if obj is None: + elif obj is None: result = tp() return result - if not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): + elif isinstance(obj, Array): + ptr = tp.__new__(tp) + ptr._buffer = tp._ffiarray(1, autofree=True) + ptr._buffer[0] = obj._buffer + result = ptr + elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) - result = tp() + else: + result = tp() + result._buffer[0] = obj._buffer[0] # The casted objects '_objects' member: - # It must certainly contain the source objects one. + # From now on, both objects will use the same dictionary + # It must certainly contain the source objects # It must contain the source object itself. if obj._ensure_objects() is not None: - result._objects = {keepalive_key(0): obj._objects, - keepalive_key(1): obj} + result._objects = obj._objects + if isinstance(obj._objects, dict): + result._objects[id(obj)] = obj - result._buffer[0] = obj._buffer[0] return result Modified: pypy/trunk/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/primitive.py (original) +++ pypy/trunk/lib_pypy/_ctypes/primitive.py Thu Dec 2 00:45:35 2010 @@ -132,8 +132,8 @@ ConvMode.errors) #self._objects = value array = _rawffi.Array('c')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -155,8 +155,8 @@ ConvMode.errors) #self._objects = value array = _rawffi.Array('u')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -174,8 +174,8 @@ def _setvalue(self, value): if isinstance(value, str): array = _rawffi.Array('c')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -271,7 +271,9 @@ def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) - return output.value + if self.__bases__[0] is _SimpleCData: + return output.value + return output def _sizeofinstances(self): return _rawffi.sizeof(self._type_) @@ -286,8 +288,12 @@ __metaclass__ = SimpleType _type_ = 'i' - def __init__(self, value=DEFAULT_VALUE): + def __new__(cls, *args, **kwds): + self = _CData.__new__(cls, *args, **kwds) self._buffer = self._ffiarray(1, autofree=True) + return self + + def __init__(self, value=DEFAULT_VALUE): if value is not DEFAULT_VALUE: self.value = value @@ -312,7 +318,11 @@ return self.value def __repr__(self): - return "%s(%r)" % (type(self).__name__, self.value) + if type(self).__bases__[0] is _SimpleCData: + return "%s(%r)" % (type(self).__name__, self.value) + else: + return "<%s object at 0x%x>" % (type(self).__name__, + id(self)) def __nonzero__(self): return self._buffer[0] not in (0, '\x00') Modified: pypy/trunk/lib_pypy/_ctypes/structure.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/structure.py (original) +++ pypy/trunk/lib_pypy/_ctypes/structure.py Thu Dec 2 00:45:35 2010 @@ -34,9 +34,11 @@ if not isinstance(tp, _CDataMeta): raise TypeError("Expected CData subclass, got %s" % (tp,)) import ctypes - all_fields = _fields_[:] - for cls in inspect.getmro(superclass): - all_fields += getattr(cls, '_fields_', []) + all_fields = [] + for cls in reversed(inspect.getmro(superclass)): + # The first field comes from the most base class + all_fields.extend(getattr(cls, '_fields_', [])) + all_fields.extend(_fields_) names = [name for name, ctype in all_fields] rawfields = [(name, ctype._ffishape) for name, ctype in all_fields] @@ -168,7 +170,7 @@ def __init__(self, *args, **kwds): if len(args) > len(self._names): - raise TypeError("too many arguments") + raise TypeError("too many initializers") for name, arg in zip(self._names, args): if name in kwds: raise TypeError("duplicate value for argument %r" % ( Modified: pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py ============================================================================== --- pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py (original) +++ pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py Thu Dec 2 00:45:35 2010 @@ -11,21 +11,23 @@ py.test.skip("pypy white-box test") from _ctypes.function import CFuncPtr - guess = CFuncPtr._guess_argtypes + def guess(value): + cobj = CFuncPtr._conv_param(None, value, 0) + return type(cobj) - assert guess([13]) == [c_int] - assert guess([0]) == [c_int] - assert guess(['xca']) == [c_char_p] - assert guess([None]) == [c_void_p] - assert guess([c_int(3)]) == [c_int] - assert guess([u'xca']) == [c_wchar_p] + assert guess(13) == c_int + assert guess(0) == c_int + assert guess('xca') == c_char_p + assert guess(None) == c_void_p + assert guess(c_int(3)) == c_int + assert guess(u'xca') == c_wchar_p class Stuff: pass s = Stuff() s._as_parameter_ = None - assert guess([s]) == [c_void_p] + assert guess(s) == c_void_p def test_guess_unicode(): if not hasattr(sys, 'pypy_translation_info') and sys.platform != 'win32': Modified: pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py ============================================================================== --- pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py (original) +++ pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py Thu Dec 2 00:45:35 2010 @@ -99,7 +99,7 @@ def test_primitive(self): if not hasattr(sys, 'pypy_translation_info'): py.test.skip("pypy white-box test") - assert c_char_p("abc")._objects['0']._buffer[0] == "a" + assert c_char_p("abc")._objects._buffer[0] == "a" assert c_int(3)._objects is None def test_pointer_to_pointer(self): @@ -123,7 +123,7 @@ pass cf = CFUNCTYPE(c_int, c_int)(f) p1 = cast(cf, c_void_p) - assert p1._objects == {'1': cf, '0': {'0': cf}} + assert p1._objects == {id(cf): cf, '0': cf} def test_array_of_struct_with_pointer(self): class S(Structure): @@ -221,7 +221,7 @@ import gc; gc.collect() print 'x =', repr(x) assert x.value == 'hellohello' - assert x._objects.keys() == ['0'] + assert x._objects == 'hellohello' # class datum(Structure): _fields_ = [ From afa at codespeak.net Thu Dec 2 00:50:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 2 Dec 2010 00:50:11 +0100 (CET) Subject: [pypy-svn] r79729 - in pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils: . command tests Message-ID: <20101201235011.C0143282BD4@codespeak.net> Author: afa Date: Thu Dec 2 00:50:09 2010 New Revision: 79729 Added: pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/ - copied from r79593, pypy/branch/fast-forward/lib-python/2.7.0/distutils/ pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/README - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/README pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/__init__.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/__init__.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/archive_util.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/archive_util.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/bcppcompiler.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/bcppcompiler.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/ccompiler.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/ccompiler.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/cmd.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/cmd.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/command/ - copied from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/command/ pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/config.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/config.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/core.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/core.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/cygwinccompiler.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/cygwinccompiler.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/debug.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/debug.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/dep_util.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/dep_util.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/dir_util.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/dir_util.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/dist.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/dist.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/emxccompiler.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/emxccompiler.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/errors.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/errors.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/extension.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/extension.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/fancy_getopt.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/fancy_getopt.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/file_util.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/file_util.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/filelist.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/filelist.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/log.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/log.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/msvc9compiler.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/msvc9compiler.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/msvccompiler.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/msvccompiler.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/spawn.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/spawn.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/sysconfig.py - copied, changed from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/sysconfig.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/sysconfig_cpython.py - copied unchanged from r79593, pypy/branch/fast-forward/lib-python/2.7.0/distutils/sysconfig.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/sysconfig_pypy.py - copied, changed from r79593, pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/tests/ - copied from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/tests/ pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/text_file.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/text_file.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/unixccompiler.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/unixccompiler.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/util.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/util.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/version.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/version.py pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/versionpredicate.py - copied unchanged from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/versionpredicate.py Log: Customize sysconfig.py like 2.5.2 Copied: pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/sysconfig.py (from r79700, pypy/branch/fast-forward/lib-python/2.7.0/distutils/sysconfig.py) ============================================================================== --- pypy/branch/fast-forward/lib-python/2.7.0/distutils/sysconfig.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/sysconfig.py Thu Dec 2 00:50:09 2010 @@ -11,587 +11,17 @@ __revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $" -import os -import re -import string import sys -from distutils.errors import DistutilsPlatformError -# These are needed in a couple of spots, so just compute them once. -PREFIX = os.path.normpath(sys.prefix) -EXEC_PREFIX = os.path.normpath(sys.exec_prefix) +# The content of this file is redirected from +# sysconfig_cpython or sysconfig_pypy. -# Path to the base directory of the project. On Windows the binary may -# live in project/PCBuild9. If we're dealing with an x64 Windows build, -# it'll live in project/PCbuild/amd64. -project_base = os.path.dirname(os.path.abspath(sys.executable)) -if os.name == "nt" and "pcbuild" in project_base[-8:].lower(): - project_base = os.path.abspath(os.path.join(project_base, os.path.pardir)) -# PC/VS7.1 -if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower(): - project_base = os.path.abspath(os.path.join(project_base, os.path.pardir, - os.path.pardir)) -# PC/AMD64 -if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower(): - project_base = os.path.abspath(os.path.join(project_base, os.path.pardir, - os.path.pardir)) +if '__pypy__' in sys.builtin_module_names: + from distutils.sysconfig_pypy import * + from distutils.sysconfig_pypy import _config_vars # needed by setuptools +else: + from distutils.sysconfig_cpython import * + from distutils.sysconfig_pypy import _config_vars # needed by setuptools -# python_build: (Boolean) if true, we're either building Python or -# building an extension with an un-installed Python, so we use -# different (hard-wired) directories. -# Setup.local is available for Makefile builds including VPATH builds, -# Setup.dist is available on Windows -def _python_build(): - for fn in ("Setup.dist", "Setup.local"): - if os.path.isfile(os.path.join(project_base, "Modules", fn)): - return True - return False -python_build = _python_build() - -def get_python_version(): - """Return a string containing the major and minor Python version, - leaving off the patchlevel. Sample return values could be '1.5' - or '2.2'. - """ - return sys.version[:3] - - -def get_python_inc(plat_specific=0, prefix=None): - """Return the directory containing installed Python header files. - - If 'plat_specific' is false (the default), this is the path to the - non-platform-specific header files, i.e. Python.h and so on; - otherwise, this is the path to platform-specific header files - (namely pyconfig.h). - - If 'prefix' is supplied, use it instead of sys.prefix or - sys.exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - prefix = plat_specific and EXEC_PREFIX or PREFIX - - if os.name == "posix": - if python_build: - buildir = os.path.dirname(sys.executable) - if plat_specific: - # python.h is located in the buildir - inc_dir = buildir - else: - # the source dir is relative to the buildir - srcdir = os.path.abspath(os.path.join(buildir, - get_config_var('srcdir'))) - # Include is located in the srcdir - inc_dir = os.path.join(srcdir, "Include") - return inc_dir - return os.path.join(prefix, "include", "python" + get_python_version()) - elif os.name == "nt": - return os.path.join(prefix, "include") - elif os.name == "os2": - return os.path.join(prefix, "Include") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its C header files " - "on platform '%s'" % os.name) - - -def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): - """Return the directory containing the Python library (standard or - site additions). - - If 'plat_specific' is true, return the directory containing - platform-specific modules, i.e. any module from a non-pure-Python - module distribution; otherwise, return the platform-shared library - directory. If 'standard_lib' is true, return the directory - containing standard Python library modules; otherwise, return the - directory for site-specific modules. - - If 'prefix' is supplied, use it instead of sys.prefix or - sys.exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - prefix = plat_specific and EXEC_PREFIX or PREFIX - - if os.name == "posix": - libpython = os.path.join(prefix, - "lib", "python" + get_python_version()) - if standard_lib: - return libpython - else: - return os.path.join(libpython, "site-packages") - - elif os.name == "nt": - if standard_lib: - return os.path.join(prefix, "Lib") - else: - if get_python_version() < "2.2": - return prefix - else: - return os.path.join(prefix, "Lib", "site-packages") - - elif os.name == "os2": - if standard_lib: - return os.path.join(prefix, "Lib") - else: - return os.path.join(prefix, "Lib", "site-packages") - - else: - raise DistutilsPlatformError( - "I don't know where Python installs its library " - "on platform '%s'" % os.name) - - -def customize_compiler(compiler): - """Do any platform-specific customization of a CCompiler instance. - - Mainly needed on Unix, so we can plug in the information that - varies across Unices and is stored in Python's Makefile. - """ - if compiler.compiler_type == "unix": - (cc, cxx, opt, cflags, ccshared, ldshared, so_ext) = \ - get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS', - 'CCSHARED', 'LDSHARED', 'SO') - - if 'CC' in os.environ: - cc = os.environ['CC'] - if 'CXX' in os.environ: - cxx = os.environ['CXX'] - if 'LDSHARED' in os.environ: - ldshared = os.environ['LDSHARED'] - if 'CPP' in os.environ: - cpp = os.environ['CPP'] - else: - cpp = cc + " -E" # not always - if 'LDFLAGS' in os.environ: - ldshared = ldshared + ' ' + os.environ['LDFLAGS'] - if 'CFLAGS' in os.environ: - cflags = opt + ' ' + os.environ['CFLAGS'] - ldshared = ldshared + ' ' + os.environ['CFLAGS'] - if 'CPPFLAGS' in os.environ: - cpp = cpp + ' ' + os.environ['CPPFLAGS'] - cflags = cflags + ' ' + os.environ['CPPFLAGS'] - ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] - - cc_cmd = cc + ' ' + cflags - compiler.set_executables( - preprocessor=cpp, - compiler=cc_cmd, - compiler_so=cc_cmd + ' ' + ccshared, - compiler_cxx=cxx, - linker_so=ldshared, - linker_exe=cc) - - compiler.shared_lib_extension = so_ext - - -def get_config_h_filename(): - """Return full pathname of installed pyconfig.h file.""" - if python_build: - if os.name == "nt": - inc_dir = os.path.join(project_base, "PC") - else: - inc_dir = project_base - else: - inc_dir = get_python_inc(plat_specific=1) - if get_python_version() < '2.2': - config_h = 'config.h' - else: - # The name of the config.h file changed in 2.2 - config_h = 'pyconfig.h' - return os.path.join(inc_dir, config_h) - - -def get_makefile_filename(): - """Return full pathname of installed Makefile from the Python build.""" - if python_build: - return os.path.join(os.path.dirname(sys.executable), "Makefile") - lib_dir = get_python_lib(plat_specific=1, standard_lib=1) - return os.path.join(lib_dir, "config", "Makefile") - - -def parse_config_h(fp, g=None): - """Parse a config.h-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - if g is None: - g = {} - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - # - while 1: - line = fp.readline() - if not line: - break - m = define_rx.match(line) - if m: - n, v = m.group(1, 2) - try: v = int(v) - except ValueError: pass - g[n] = v - else: - m = undef_rx.match(line) - if m: - g[m.group(1)] = 0 - return g - - -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") -_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)") -_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}") - -def parse_makefile(fn, g=None): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - from distutils.text_file import TextFile - fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1) - - if g is None: - g = {} - done = {} - notdone = {} - - while 1: - line = fp.readline() - if line is None: # eof - break - m = _variable_rx.match(line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # do variable interpolation here - while notdone: - for name in notdone.keys(): - value = notdone[name] - m = _findvar1_rx.search(value) or _findvar2_rx.search(value) - if m: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - else: - done[n] = item = "" - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - del notdone[name] - else: - # bogus variable reference; just drop it since we can't deal - del notdone[name] - - fp.close() - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - g.update(done) - return g - - -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. - """ - - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. - - while 1: - m = _findvar1_rx.search(s) or _findvar2_rx.search(s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] - else: - break - return s - - -_config_vars = None - -def _init_posix(): - """Initialize the module as appropriate for POSIX systems.""" - g = {} - # load the installed Makefile: - try: - filename = get_makefile_filename() - parse_makefile(filename, g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # load the installed pyconfig.h: - try: - filename = get_config_h_filename() - parse_config_h(file(filename), g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # On MacOSX we need to check the setting of the environment variable - # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so - # it needs to be compatible. - # If it isn't set we set it to the configure-time value - if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g: - cfg_target = g['MACOSX_DEPLOYMENT_TARGET'] - cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '') - if cur_target == '': - cur_target = cfg_target - os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target) - elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')): - my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure' - % (cur_target, cfg_target)) - raise DistutilsPlatformError(my_msg) - - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if python_build: - g['LDSHARED'] = g['BLDSHARED'] - - elif get_python_version() < '2.1': - # The following two branches are for 1.5.2 compatibility. - if sys.platform == 'aix4': # what about AIX 3.x ? - # Linker script is in the config directory, not in Modules as the - # Makefile says. - python_lib = get_python_lib(standard_lib=1) - ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix') - python_exp = os.path.join(python_lib, 'config', 'python.exp') - - g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp) - - elif sys.platform == 'beos': - # Linker script is in the config directory. In the Makefile it is - # relative to the srcdir, which after installation no longer makes - # sense. - python_lib = get_python_lib(standard_lib=1) - linkerscript_path = string.split(g['LDSHARED'])[0] - linkerscript_name = os.path.basename(linkerscript_path) - linkerscript = os.path.join(python_lib, 'config', - linkerscript_name) - - # XXX this isn't the right place to do this: adding the Python - # library to the link, if needed, should be in the "build_ext" - # command. (It's also needed for non-MS compilers on Windows, and - # it's taken care of for them by the 'build_ext.get_libraries()' - # method.) - g['LDSHARED'] = ("%s -L%s/lib -lpython%s" % - (linkerscript, PREFIX, get_python_version())) - - global _config_vars - _config_vars = g - - -def _init_nt(): - """Initialize the module as appropriate for NT""" - g = {} - # set basic install directories - g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) - g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) - - # XXX hmmm.. a normal install puts include files here - g['INCLUDEPY'] = get_python_inc(plat_specific=0) - - g['SO'] = '.pyd' - g['EXE'] = ".exe" - g['VERSION'] = get_python_version().replace(".", "") - g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) - - global _config_vars - _config_vars = g - - -def _init_mac(): - """Initialize the module as appropriate for Macintosh systems""" - g = {} - # set basic install directories - g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) - g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) - - # XXX hmmm.. a normal install puts include files here - g['INCLUDEPY'] = get_python_inc(plat_specific=0) - - import MacOS - if not hasattr(MacOS, 'runtimemodel'): - g['SO'] = '.ppc.slb' - else: - g['SO'] = '.%s.slb' % MacOS.runtimemodel - - # XXX are these used anywhere? - g['install_lib'] = os.path.join(EXEC_PREFIX, "Lib") - g['install_platlib'] = os.path.join(EXEC_PREFIX, "Mac", "Lib") - - # These are used by the extension module build - g['srcdir'] = ':' - global _config_vars - _config_vars = g - - -def _init_os2(): - """Initialize the module as appropriate for OS/2""" - g = {} - # set basic install directories - g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) - g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) - - # XXX hmmm.. a normal install puts include files here - g['INCLUDEPY'] = get_python_inc(plat_specific=0) - - g['SO'] = '.pyd' - g['EXE'] = ".exe" - - global _config_vars - _config_vars = g - - -def get_config_vars(*args): - """With no arguments, return a dictionary of all configuration - variables relevant for the current platform. Generally this includes - everything needed to build extensions and install both pure modules and - extensions. On Unix, this means every variable defined in Python's - installed Makefile; on Windows and Mac OS it's a much smaller set. - - With arguments, return a list of values that result from looking up - each argument in the configuration variable dictionary. - """ - global _config_vars - if _config_vars is None: - func = globals().get("_init_" + os.name) - if func: - func() - else: - _config_vars = {} - - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _config_vars['prefix'] = PREFIX - _config_vars['exec_prefix'] = EXEC_PREFIX - - if sys.platform == 'darwin': - kernel_version = os.uname()[2] # Kernel version (8.4.3) - major_version = int(kernel_version.split('.')[0]) - - if major_version < 8: - # On Mac OS X before 10.4, check if -arch and -isysroot - # are in CFLAGS or LDFLAGS and remove them if they are. - # This is needed when building extensions on a 10.3 system - # using a universal build of python. - for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED', - # a number of derived variables. These need to be - # patched up as well. - 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): - flags = _config_vars[key] - flags = re.sub('-arch\s+\w+\s', ' ', flags) - flags = re.sub('-isysroot [^ \t]*', ' ', flags) - _config_vars[key] = flags - - else: - - # Allow the user to override the architecture flags using - # an environment variable. - # NOTE: This name was introduced by Apple in OSX 10.5 and - # is used by several scripting languages distributed with - # that OS release. - - if 'ARCHFLAGS' in os.environ: - arch = os.environ['ARCHFLAGS'] - for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED', - # a number of derived variables. These need to be - # patched up as well. - 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): - - flags = _config_vars[key] - flags = re.sub('-arch\s+\w+\s', ' ', flags) - flags = flags + ' ' + arch - _config_vars[key] = flags - - # If we're on OSX 10.5 or later and the user tries to - # compiles an extension using an SDK that is not present - # on the current machine it is better to not use an SDK - # than to fail. - # - # The major usecase for this is users using a Python.org - # binary installer on OSX 10.6: that installer uses - # the 10.4u SDK, but that SDK is not installed by default - # when you install Xcode. - # - m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS']) - if m is not None: - sdk = m.group(1) - if not os.path.exists(sdk): - for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED', - # a number of derived variables. These need to be - # patched up as well. - 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): - - flags = _config_vars[key] - flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags) - _config_vars[key] = flags - - if args: - vals = [] - for name in args: - vals.append(_config_vars.get(name)) - return vals - else: - return _config_vars - -def get_config_var(name): - """Return the value of a single variable using the dictionary - returned by 'get_config_vars()'. Equivalent to - get_config_vars().get(name) - """ - return get_config_vars().get(name) Copied: pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/sysconfig_pypy.py (from r79593, pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py) ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/sysconfig_pypy.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/sysconfig_pypy.py Thu Dec 2 00:50:09 2010 @@ -108,8 +108,10 @@ """ return get_config_vars().get(name) -def customize_compiler(options): +def customize_compiler(compiler): """Dummy method to let some easy_install packages that have optional C speedup components. """ - pass + if compiler.compiler_type == "unix": + compiler.compiler_so.extend(['-fPIC', '-Wimplicit']) + compiler.shared_lib_extension = get_config_var('SO') From afa at codespeak.net Thu Dec 2 00:55:30 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 2 Dec 2010 00:55:30 +0100 (CET) Subject: [pypy-svn] r79730 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101201235530.2BE7A282BDD@codespeak.net> Author: afa Date: Thu Dec 2 00:55:28 2010 New Revision: 79730 Added: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_copy.py - copied, changed from r79700, pypy/branch/fast-forward/lib-python/2.7.0/test/test_copy.py Log: Some gc_collect() to clear weakrefs and make the tests pass Copied: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_copy.py (from r79700, pypy/branch/fast-forward/lib-python/2.7.0/test/test_copy.py) ============================================================================== --- pypy/branch/fast-forward/lib-python/2.7.0/test/test_copy.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_copy.py Thu Dec 2 00:55:28 2010 @@ -637,6 +637,7 @@ self.assertEqual(v[c], d) self.assertEqual(len(v), 2) del c, d + test_support.gc_collect() self.assertEqual(len(v), 1) x, y = C(), C() # The underlying containers are decoupled @@ -666,6 +667,7 @@ self.assertEqual(v[a].i, b.i) self.assertEqual(v[c].i, d.i) del c + test_support.gc_collect() self.assertEqual(len(v), 1) def test_deepcopy_weakvaluedict(self): @@ -689,6 +691,7 @@ self.assertTrue(t is d) del x, y, z, t del d + test_support.gc_collect() self.assertEqual(len(v), 1) def test_deepcopy_bound_method(self): From dan at codespeak.net Thu Dec 2 03:27:38 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Thu, 2 Dec 2010 03:27:38 +0100 (CET) Subject: [pypy-svn] r79731 - pypy/branch/psycopg2compatibility/pypy/module/cpyext/src Message-ID: <20101202022738.B68CC282B90@codespeak.net> Author: dan Date: Thu Dec 2 03:27:35 2010 New Revision: 79731 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/src/modsupport.c Log: Such an innocuous change made all the difference. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/src/modsupport.c ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/src/modsupport.c (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/src/modsupport.c Thu Dec 2 03:27:35 2010 @@ -615,8 +615,8 @@ if (result != NULL && n > 0) { for (i = 0; i < n; ++i) { tmp = (PyObject *)va_arg(va, PyObject *); - PyTuple_SET_ITEM(result, i, tmp); Py_INCREF(tmp); + PyTuple_SET_ITEM(result, i, tmp); } } return result; From dan at codespeak.net Thu Dec 2 03:28:34 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Thu, 2 Dec 2010 03:28:34 +0100 (CET) Subject: [pypy-svn] r79732 - pypy/branch/psycopg2compatibility/pypy/module/cpyext Message-ID: <20101202022834.6806B282B9D@codespeak.net> Author: dan Date: Thu Dec 2 03:28:32 2010 New Revision: 79732 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/api.py Log: Really uglied up the place... (Added logging to "case 3" CPyExt calls. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/api.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/api.py Thu Dec 2 03:28:32 2010 @@ -33,7 +33,6 @@ from py.builtin import BaseException from pypy.tool.sourcetools import func_with_new_name from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rlib.debug import debug_print DEBUG_WRAPPER = True @@ -42,18 +41,56 @@ from pypy.tool.call_logger import CallLogger, SkipArgument from pypy.rlib.objectmodel import specialize +def debug_func(category): + def decorator(f): + from pypy.rlib.debug import debug_start, debug_stop + @wraps(f) + def debugged(*args): + debug_start(category) + result = f(*args) + debug_stop(category) + return result + return debugged + return decorator + class CPyExtCallLogger(object): def __init__(self): self.indentation = 0 + def get_indent_string(self, depth=None): + if depth is None: + depth = self.indentation + + return ' ' * depth + def log(self, logstr, depth=0): - debug_print((' ' * depth) + logstr) + from pypy.rlib.debug import debug_print + debug_print(self.get_indent_string() + logstr) + + def log_cpyext_rpython_call(self, f): + if DEBUG_WRAPPER: + name = f.__name__ + #@debug_func('cpyext') + @wraps(f) + def wrapped(space, *args): + argstrs = [repr(x) for x in args] + argstr = ', '.join(argstrs) + + self.log("%s(%s)" % (name, argstr), depth=self.indentation) + self.indentation += 1 + result = f(space, *args) + self.indentation -= 1 + self.log("%s(%s)->%s" % (name, argstr, repr(result)), depth=self.indentation) + return result + return wrapped + else: + return f def log_cpyext_call(self, space, f): if DEBUG_WRAPPER: from pypy.module.cpyext.pyobject import make_ref, from_ref from pypy.module.cpyext.object import PyObject_Repr - from pypy.module.cpyext.pyobject import Py_DecRef + from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef api_function = f.api_func argtypes = api_function.argtypes result_type = api_function.restype @@ -62,17 +99,22 @@ argtypes_enum_ui = unrolling_iterable(enumerate(zip(api_function.argtypes, [name.startswith("w_") for name in api_function.argnames]), start=1)) name = f.__name__ # XXX: above line was overwriting name "name" + + #@debug_func('cpyext') @wraps(f) def wrapped(*args): argstrs = [] for i, (argtype, is_wrapped) in argtypes_enum_ui: arg = args[i] if is_PyObject(argtype) and is_wrapped: - if arg is not None: - pyrepr = PyObject_Repr(space, arg) - argstrs.append(space.str_w(pyrepr)) + w_arg = arg + if w_arg is not None: + Py_IncRef(space, w_arg) + py_repr = PyObject_Repr(space, w_arg) + Py_DecRef(space, w_arg) + argstrs.append(space.str_w(py_repr)) else: - argstrs.append(repr(arg)) + argstrs.append(repr(w_arg)) else: argstrs.append(repr(arg)) argstr = ', '.join(argstrs) @@ -90,7 +132,9 @@ else: return f -log_call = CPyExtCallLogger().log_cpyext_call +cpy_logger = CPyExtCallLogger() +log_call = cpy_logger.log_cpyext_call +rlog_call = cpy_logger.log_cpyext_rpython_call # update these for other platforms Py_ssize_t = lltype.Signed @@ -278,7 +322,10 @@ zip(api_function.argtypes, [tp_name.startswith("w_") for tp_name in names]))) + logged_func = rlog_call(func) + @specialize.ll() + @wraps(func) def unwrapper(space, *args): from pypy.module.cpyext.pyobject import Py_DecRef from pypy.module.cpyext.pyobject import make_ref, from_ref @@ -312,7 +359,10 @@ newargs += (arg, ) try: try: - res = func(space, *newargs) + if not we_are_translated() and DEBUG_WRAPPER: + res = logged_func(space, *newargs) + else: + res = func(space, *newargs) except OperationError, e: if not catch_exception: raise @@ -757,7 +807,7 @@ py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) typedescr.attach(space, py_obj, w_obj) - track_reference(space, py_obj, w_obj) + track_reference(space, py_obj, w_obj, replace=True) else: assert False, "Unknown static object: %s %s" % (typ, name) From dan at codespeak.net Thu Dec 2 03:30:52 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Thu, 2 Dec 2010 03:30:52 +0100 (CET) Subject: [pypy-svn] r79733 - pypy/branch/psycopg2compatibility/pypy/module/cpyext Message-ID: <20101202023052.4C290282BD4@codespeak.net> Author: dan Date: Thu Dec 2 03:30:50 2010 New Revision: 79733 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py Log: Beat the refcount logging code until it looked pretty in conjunction with the rest of CPyExt logging. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py Thu Dec 2 03:30:50 2010 @@ -250,13 +250,16 @@ class InvalidPointerException(Exception): pass -DEBUG_REFCOUNT = False +DEBUG_REFCOUNT = True def debug_refcount(*args, **kwargs): + from pypy.module.cpyext.api import cpy_logger frame_stackdepth = kwargs.pop("frame_stackdepth", 2) assert not kwargs frame = sys._getframe(frame_stackdepth) - print >>sys.stderr, "%25s" % (frame.f_code.co_name, ), + sys.stderr.write(cpy_logger.get_indent_string()) + sys.stderr.write(frame.f_code.co_name) + sys.stderr.write(' ') for arg in args: print >>sys.stderr, arg, print >>sys.stderr @@ -291,7 +294,10 @@ if DEBUG_REFCOUNT: debug_refcount("MAKREF", py_obj, w_obj) if not replace: - assert w_obj not in state.py_objects_w2r + if not we_are_translated(): + assert w_obj not in state.py_objects_w2r, "%r shouldn't be tracked already" % w_obj + else: + assert w_obj not in state.py_objects_w2r assert ptr not in state.py_objects_r2w assert ptr not in state.borrowed_objects state.py_objects_w2r[w_obj] = py_obj From afa at codespeak.net Thu Dec 2 08:51:28 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 2 Dec 2010 08:51:28 +0100 (CET) Subject: [pypy-svn] r79734 - pypy/branch/fast-forward/pypy/module/cpyext/src Message-ID: <20101202075128.92979282B90@codespeak.net> Author: afa Date: Thu Dec 2 08:51:24 2010 New Revision: 79734 Modified: pypy/branch/fast-forward/pypy/module/cpyext/src/structseq.c Log: The reminder "worked", this code is not needed with 2.7 Modified: pypy/branch/fast-forward/pypy/module/cpyext/src/structseq.c ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/src/structseq.c (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/src/structseq.c Thu Dec 2 08:51:24 2010 @@ -5,13 +5,6 @@ #include "structmember.h" #include "structseq.h" -#ifdef Py_TYPE -#error "Please remove these definitions" -#else -#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) -#endif - static char visible_length_key[] = "n_sequence_fields"; static char real_length_key[] = "n_fields"; static char unnamed_fields_key[] = "n_unnamed_fields"; From fijal at codespeak.net Thu Dec 2 10:13:01 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 10:13:01 +0100 (CET) Subject: [pypy-svn] r79735 - in pypy/branch/out-of-line-guards/pypy/jit/codewriter: . test Message-ID: <20101202091301.0C4E1282B9D@codespeak.net> Author: fijal Date: Thu Dec 2 10:13:00 2010 New Revision: 79735 Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py Log: Implement jtransform layer of jit_invariant_field Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py Thu Dec 2 10:13:00 2010 @@ -14,6 +14,8 @@ from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import varoftype +class JitInvariantError(Exception): + pass def transform_graph(graph, cpu=None, callcontrol=None, portal_jd=None): """Transform a control flow graph to make it suitable for @@ -536,7 +538,11 @@ if immut == "[*]": self.immutable_arrays[op.result] = True else: - pure = '' + jit_inv = v_inst.concretetype.TO._hints.get('jit_invariant_fields') + if jit_inv and op.args[1].value[len('inst_'):] in jit_inv.fields: + pure = '_pure' + else: + pure = '' argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') descr = self.cpu.fielddescrof(v_inst.concretetype.TO, c_fieldname.value) @@ -560,6 +566,10 @@ return [SpaceOperation('-live-', [], None), SpaceOperation('setfield_vable_%s' % kind, [v_inst, descr, v_value], None)] + jit_inv = v_inst.concretetype.TO._hints.get('jit_invariant_fields') + if jit_inv and op.args[1].value[len('inst_'):] in jit_inv.fields: + raise JitInvariantError("setfield on jit invariant should not" + " be ssen by jit") argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') descr = self.cpu.fielddescrof(v_inst.concretetype.TO, c_fieldname.value) Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py Thu Dec 2 10:13:00 2010 @@ -2,7 +2,7 @@ import random from pypy.objspace.flow.model import FunctionGraph, Block, Link from pypy.objspace.flow.model import SpaceOperation, Variable, Constant -from pypy.jit.codewriter.jtransform import Transformer +from pypy.jit.codewriter.jtransform import Transformer, JitInvariantError from pypy.jit.metainterp.history import getkind from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist from pypy.translator.unsimplify import varoftype @@ -942,3 +942,34 @@ assert op1.args[1] == 'calldescr-%d' % effectinfo.EffectInfo.OS_ARRAYCOPY assert op1.args[2] == ListOfKind('int', [v3, v4, v5]) assert op1.args[3] == ListOfKind('ref', [v1, v2]) + +def test_jit_invariant(): + from pypy.rpython.rclass import FieldListAccessor + from pypy.rpython.lltypesystem.rclass import ASMCODE + accessor = FieldListAccessor() + accessor.initialize(None, {'x': 'asmcodes_x'}) + v2 = varoftype(lltype.Signed) + STRUCT = lltype.GcStruct('struct', ('inst_x', lltype.Signed), + ('asmcodes_x', lltype.Ptr(ASMCODE)), + hints={'jit_invariant_fields': accessor}) + op = SpaceOperation('getfield', [const(lltype.malloc(STRUCT)), + Constant('inst_x', lltype.Void)], v2) + tr = Transformer(FakeCPU()) + op1 = tr.rewrite_operation(op) + assert op1.opname == 'getfield_gc_i_pure' + +def test_jit_invariant_setfield(): + from pypy.rpython.rclass import FieldListAccessor + from pypy.rpython.lltypesystem.rclass import ASMCODE + accessor = FieldListAccessor() + accessor.initialize(None, {'x': 'asmcodes_x'}) + v1 = varoftype(lltype.Signed) + STRUCT = lltype.GcStruct('struct', ('inst_x', lltype.Signed), + ('asmcodes_x', lltype.Ptr(ASMCODE)), + hints={'jit_invariant_fields': accessor}) + op = SpaceOperation('setfield', [const(lltype.malloc(STRUCT)), + Constant('inst_x', lltype.Void), v1], + varoftype(lltype.Void)) + tr = Transformer(FakeCPU()) + raises(JitInvariantError, tr.rewrite_operation, op) + From fijal at codespeak.net Thu Dec 2 10:53:12 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 10:53:12 +0100 (CET) Subject: [pypy-svn] r79736 - in pypy/branch/out-of-line-guards/pypy/jit: codewriter codewriter/test metainterp/test Message-ID: <20101202095312.245A4282B90@codespeak.net> Author: fijal Date: Thu Dec 2 10:53:10 2010 New Revision: 79736 Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py Log: test and a fix Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py Thu Dec 2 10:53:10 2010 @@ -539,7 +539,7 @@ self.immutable_arrays[op.result] = True else: jit_inv = v_inst.concretetype.TO._hints.get('jit_invariant_fields') - if jit_inv and op.args[1].value[len('inst_'):] in jit_inv.fields: + if jit_inv and op.args[1].value in jit_inv.fields: pure = '_pure' else: pure = '' @@ -567,7 +567,7 @@ SpaceOperation('setfield_vable_%s' % kind, [v_inst, descr, v_value], None)] jit_inv = v_inst.concretetype.TO._hints.get('jit_invariant_fields') - if jit_inv and op.args[1].value[len('inst_'):] in jit_inv.fields: + if jit_inv and op.args[1].value in jit_inv.fields: raise JitInvariantError("setfield on jit invariant should not" " be ssen by jit") argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py Thu Dec 2 10:53:10 2010 @@ -947,7 +947,7 @@ from pypy.rpython.rclass import FieldListAccessor from pypy.rpython.lltypesystem.rclass import ASMCODE accessor = FieldListAccessor() - accessor.initialize(None, {'x': 'asmcodes_x'}) + accessor.initialize(None, {'inst_x': 'asmcodes_x'}) v2 = varoftype(lltype.Signed) STRUCT = lltype.GcStruct('struct', ('inst_x', lltype.Signed), ('asmcodes_x', lltype.Ptr(ASMCODE)), @@ -962,7 +962,7 @@ from pypy.rpython.rclass import FieldListAccessor from pypy.rpython.lltypesystem.rclass import ASMCODE accessor = FieldListAccessor() - accessor.initialize(None, {'x': 'asmcodes_x'}) + accessor.initialize(None, {'inst_x': 'asmcodes_x'}) v1 = varoftype(lltype.Signed) STRUCT = lltype.GcStruct('struct', ('inst_x', lltype.Signed), ('asmcodes_x', lltype.Ptr(ASMCODE)), Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py Thu Dec 2 10:53:10 2010 @@ -718,6 +718,24 @@ assert res == 210 self.check_operations_history(getfield_gc=0) + def test_getfield_jit_invariant(self): + class A(object): + _jit_invariant_fields_ = 'x' + + a1 = A() + a1.x = 5 + a2 = A() + a2.x = 8 + + def f(x): + if x: + a = a1 + else: + a = a2 + return a.x + res = self.interp_operations(f, [-3]) + self.check_operations_history(getfield_gc = 0) + def test_setfield_bool(self): class A: def __init__(self): From arigo at codespeak.net Thu Dec 2 11:14:55 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 2 Dec 2010 11:14:55 +0100 (CET) Subject: [pypy-svn] r79737 - pypy/trunk/lib_pypy Message-ID: <20101202101455.E6B75282B9D@codespeak.net> Author: arigo Date: Thu Dec 2 11:14:53 2010 New Revision: 79737 Modified: pypy/trunk/lib_pypy/_hashlib.py pypy/trunk/lib_pypy/_locale.py pypy/trunk/lib_pypy/_marshal.py pypy/trunk/lib_pypy/_minimal_curses.py pypy/trunk/lib_pypy/cPickle.py pypy/trunk/lib_pypy/cmath.py pypy/trunk/lib_pypy/grp.py pypy/trunk/lib_pypy/itertools.py pypy/trunk/lib_pypy/msvcrt.py pypy/trunk/lib_pypy/pwd.py pypy/trunk/lib_pypy/pyexpat.py pypy/trunk/lib_pypy/resource.py pypy/trunk/lib_pypy/syslog.py Log: Found the proper way: try except ImportError. Sorry if e.g. you had troubles with older pypy-c's. Modified: pypy/trunk/lib_pypy/_hashlib.py ============================================================================== --- pypy/trunk/lib_pypy/_hashlib.py (original) +++ pypy/trunk/lib_pypy/_hashlib.py Thu Dec 2 11:14:53 2010 @@ -148,29 +148,29 @@ return hash(ctx, name) # shortcut functions -import __pypy__ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f - at __pypy__.builtinify + at builtinify def openssl_md5(string=''): return new('md5', string) - at __pypy__.builtinify + at builtinify def openssl_sha1(string=''): return new('sha1', string) - at __pypy__.builtinify + at builtinify def openssl_sha224(string=''): return new('sha224', string) - at __pypy__.builtinify + at builtinify def openssl_sha256(string=''): return new('sha256', string) - at __pypy__.builtinify + at builtinify def openssl_sha384(string=''): return new('sha384', string) - at __pypy__.builtinify + at builtinify def openssl_sha512(string=''): return new('sha512', string) - Modified: pypy/trunk/lib_pypy/_locale.py ============================================================================== --- pypy/trunk/lib_pypy/_locale.py (original) +++ pypy/trunk/lib_pypy/_locale.py Thu Dec 2 11:14:53 2010 @@ -11,7 +11,8 @@ # load the platform-specific cache made by running locale.ctc.py from ctypes_config_cache._locale_cache import * -import __pypy__ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f # Ubuntu Gusty i386 structure @@ -160,7 +161,7 @@ ul = ''.join(ul) string.letters = ul - at __pypy__.builtinify + at builtinify def setlocale(category, locale=None): "(integer,string=None) -> string. Activates/queries locale processing." if locale: @@ -185,7 +186,7 @@ groups.append(0) return groups - at __pypy__.builtinify + at builtinify def localeconv(): "() -> dict. Returns numeric and monetary locale-specific parameters." @@ -219,7 +220,7 @@ } return result - at __pypy__.builtinify + at builtinify def strcoll(s1, s2): "string,string -> int. Compares two strings according to the locale." @@ -238,7 +239,7 @@ # Collate the strings. return _wcscoll(s1, s2) - at __pypy__.builtinify + at builtinify def strxfrm(s): "string -> string. Returns a string that behaves for cmp locale-aware." @@ -252,7 +253,7 @@ _strxfrm(buf, s, n2) return buf.value - at __pypy__.builtinify + at builtinify def getdefaultlocale(): # TODO: Port code from CPython for Windows and Mac OS raise NotImplementedError() @@ -274,31 +275,31 @@ raise ValueError("unsupported langinfo constant") if HAS_LIBINTL: - @__pypy__.builtinify + @builtinify def gettext(msg): """gettext(msg) -> string Return translation of msg.""" return _gettext(msg) - @__pypy__.builtinify + @builtinify def dgettext(domain, msg): """dgettext(domain, msg) -> string Return translation of msg in domain.""" return _dgettext(domain, msg) - @__pypy__.builtinify + @builtinify def dcgettext(domain, msg, category): """dcgettext(domain, msg, category) -> string Return translation of msg in domain and category.""" return _dcgettext(domain, msg, category) - @__pypy__.builtinify + @builtinify def textdomain(domain): """textdomain(domain) -> string Set the C library's textdomain to domain, returning the new domain.""" return _textdomain(domain) - @__pypy__.builtinify + @builtinify def bindtextdomain(domain, dir): """bindtextdomain(domain, dir) -> string Bind the C library's domain to dir.""" @@ -309,7 +310,7 @@ return dirname if HAS_BIND_TEXTDOMAIN_CODESET: - @__pypy__.builtinify + @builtinify def bind_textdomain_codeset(domain, codeset): """bind_textdomain_codeset(domain, codeset) -> string Bind the C library's domain to codeset.""" Modified: pypy/trunk/lib_pypy/_marshal.py ============================================================================== --- pypy/trunk/lib_pypy/_marshal.py (original) +++ pypy/trunk/lib_pypy/_marshal.py Thu Dec 2 11:14:53 2010 @@ -6,6 +6,10 @@ import types from _codecs import utf_8_decode, utf_8_encode +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + TYPE_NULL = '0' TYPE_NONE = 'N' TYPE_FALSE = 'F' @@ -645,15 +649,18 @@ version = 1 + at builtinify def dump(x, f, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format m = _Marshaller(f.write) m.dump(x) + at builtinify def load(f): um = _Unmarshaller(f.read) return um.load() + at builtinify def dumps(x, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format buffer = [] @@ -661,6 +668,7 @@ m.dump(x) return ''.join(buffer) + at builtinify def loads(s): um = _FastUnmarshaller(s) return um.load() Modified: pypy/trunk/lib_pypy/_minimal_curses.py ============================================================================== --- pypy/trunk/lib_pypy/_minimal_curses.py (original) +++ pypy/trunk/lib_pypy/_minimal_curses.py Thu Dec 2 11:14:53 2010 @@ -35,23 +35,24 @@ # ____________________________________________________________ -import __pypy__ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f - at __pypy__.builtinify + at builtinify def setupterm(termstr, fd): err = ctypes.c_int(0) result = clib.setupterm(termstr, fd, ctypes.byref(err)) if result == ERR: raise error("setupterm() failed (err=%d)" % err.value) - at __pypy__.builtinify + at builtinify def tigetstr(cap): result = clib.tigetstr(cap) if ctypes.cast(result, ctypes.c_void_p).value == ERR: return None return ctypes.cast(result, ctypes.c_char_p).value - at __pypy__.builtinify + at builtinify def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) if result is None: Modified: pypy/trunk/lib_pypy/cPickle.py ============================================================================== --- pypy/trunk/lib_pypy/cPickle.py (original) +++ pypy/trunk/lib_pypy/cPickle.py Thu Dec 2 11:14:53 2010 @@ -4,7 +4,10 @@ from pickle import * from pickle import __doc__, __version__, format_version, compatible_formats -import __pypy__ + +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + BadPickleGet = KeyError UnpickleableError = PicklingError @@ -32,11 +35,11 @@ def getvalue(self): return self.__f and self.__f.getvalue() - at __pypy__.builtinify + at builtinify def dump(obj, file, protocol=None): Pickler(file, protocol).dump(obj) - at __pypy__.builtinify + at builtinify def dumps(obj, protocol=None): file = StringIO() Pickler(file, protocol).dump(obj) Modified: pypy/trunk/lib_pypy/cmath.py ============================================================================== --- pypy/trunk/lib_pypy/cmath.py (original) +++ pypy/trunk/lib_pypy/cmath.py Thu Dec 2 11:14:53 2010 @@ -5,9 +5,12 @@ # much code borrowed from mathmodule.c -import math, __pypy__ +import math from math import e, pi +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + # constants _one = complex(1., 0.) @@ -24,7 +27,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def acos(x): """acos(x) @@ -33,7 +36,7 @@ return -(_prodi(log((x+(_i*sqrt((_one-(x*x)))))))) - at __pypy__.builtinify + at builtinify def acosh(x): """acosh(x) @@ -43,7 +46,7 @@ return z+z - at __pypy__.builtinify + at builtinify def asin(x): """asin(x) @@ -55,7 +58,7 @@ return -(_prodi(log((sqrt_1_minus_x_sq+_prodi(x))))) - at __pypy__.builtinify + at builtinify def asinh(x): """asinh(x) @@ -65,7 +68,7 @@ return z+z - at __pypy__.builtinify + at builtinify def atan(x): """atan(x) @@ -74,7 +77,7 @@ return _halfi*log(((_i+x)/(_i-x))) - at __pypy__.builtinify + at builtinify def atanh(x): """atanh(x) @@ -83,7 +86,7 @@ return _half*log((_one+x)/(_one-x)) - at __pypy__.builtinify + at builtinify def cos(x): """cos(x) @@ -95,7 +98,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def cosh(x): """cosh(x) @@ -107,7 +110,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def exp(x): """exp(x) @@ -120,7 +123,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def log(x, base=None): """log(x) @@ -135,7 +138,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def log10(x): """log10(x) @@ -148,7 +151,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def sin(x): """sin(x) @@ -160,7 +163,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def sinh(x): """sinh(x) @@ -172,7 +175,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def sqrt(x): """sqrt(x) @@ -198,7 +201,7 @@ _sqrt_half = sqrt(_half) - at __pypy__.builtinify + at builtinify def tan(x): """tan(x) @@ -219,7 +222,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def tanh(x): """tanh(x) Modified: pypy/trunk/lib_pypy/grp.py ============================================================================== --- pypy/trunk/lib_pypy/grp.py (original) +++ pypy/trunk/lib_pypy/grp.py Thu Dec 2 11:14:53 2010 @@ -2,13 +2,17 @@ """ This module provides ctypes version of cpython's grp module """ -import sys, __pypy__ +import sys if sys.platform == 'win32': raise ImportError("No grp module on Windows") from ctypes import Structure, c_char_p, c_int, POINTER from ctypes_support import standard_c_lib as libc +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + gid_t = c_int class GroupStruct(Structure): @@ -64,7 +68,7 @@ return Group(res.contents.gr_name, res.contents.gr_passwd, res.contents.gr_gid, mem) - at __pypy__.builtinify + at builtinify def getgrgid(gid): res = libc.getgrgid(gid) if not res: @@ -72,7 +76,7 @@ raise KeyError(gid) return _group_from_gstruct(res) - at __pypy__.builtinify + at builtinify def getgrnam(name): if not isinstance(name, str): raise TypeError("expected string") @@ -81,7 +85,7 @@ raise KeyError(name) return _group_from_gstruct(res) - at __pypy__.builtinify + at builtinify def getgrall(): libc.setgrent() lst = [] Modified: pypy/trunk/lib_pypy/itertools.py ============================================================================== --- pypy/trunk/lib_pypy/itertools.py (original) +++ pypy/trunk/lib_pypy/itertools.py Thu Dec 2 11:14:53 2010 @@ -27,7 +27,8 @@ 'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap', 'takewhile', 'tee'] -import __pypy__ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f class chain(object): @@ -568,7 +569,7 @@ return self - at __pypy__.builtinify + at builtinify def tee(iterable, n=2): """Return n independent iterators from a single iterable. Note : once tee() has made a split, the original iterable Modified: pypy/trunk/lib_pypy/msvcrt.py ============================================================================== --- pypy/trunk/lib_pypy/msvcrt.py (original) +++ pypy/trunk/lib_pypy/msvcrt.py Thu Dec 2 11:14:53 2010 @@ -11,13 +11,16 @@ from ctypes_support import standard_c_lib as _c from ctypes_support import get_errno import errno -import __pypy__ try: open_osfhandle = _c._open_osfhandle except AttributeError: # we are not on windows raise ImportError +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int] open_osfhandle.restype = ctypes.c_int @@ -35,7 +38,7 @@ _locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] _locking.restype = ctypes.c_int - at __pypy__.builtinify + at builtinify def locking(fd, mode, nbytes): '''lock or unlock a number of bytes in a file.''' rv = _locking(fd, mode, nbytes) Modified: pypy/trunk/lib_pypy/pwd.py ============================================================================== --- pypy/trunk/lib_pypy/pwd.py (original) +++ pypy/trunk/lib_pypy/pwd.py Thu Dec 2 11:14:53 2010 @@ -10,13 +10,17 @@ exception is raised if the entry asked for cannot be found. """ -import sys, __pypy__ +import sys if sys.platform == 'win32': raise ImportError("No pwd module on Windows") from ctypes_support import standard_c_lib as libc from ctypes import Structure, POINTER, c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + uid_t = c_int gid_t = c_int @@ -79,12 +83,12 @@ _endpwent.argtypes = None _endpwent.restype = None - at __pypy__.builtinify + at builtinify def mkpwent(pw): pw = pw.contents return struct_passwd(pw) - at __pypy__.builtinify + at builtinify def getpwuid(uid): """ getpwuid(uid) -> (pw_name,pw_passwd,pw_uid, @@ -97,7 +101,7 @@ raise KeyError("getpwuid(): uid not found: %s" % uid) return mkpwent(pw) - at __pypy__.builtinify + at builtinify def getpwnam(name): """ getpwnam(name) -> (pw_name,pw_passwd,pw_uid, @@ -112,7 +116,7 @@ raise KeyError("getpwname(): name not found: %s" % name) return mkpwent(pw) - at __pypy__.builtinify + at builtinify def getpwall(): """ getpwall() -> list_of_entries Modified: pypy/trunk/lib_pypy/pyexpat.py ============================================================================== --- pypy/trunk/lib_pypy/pyexpat.py (original) +++ pypy/trunk/lib_pypy/pyexpat.py Thu Dec 2 11:14:53 2010 @@ -2,11 +2,14 @@ import ctypes import ctypes.util from ctypes import c_char_p, c_int, c_void_p, POINTER, c_char, c_wchar_p -import sys, __pypy__ +import sys # load the platform-specific cache made by running pyexpat.ctc.py from ctypes_config_cache._pyexpat_cache import * +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + lib = ctypes.CDLL(ctypes.util.find_library('expat')) @@ -425,11 +428,11 @@ new_parser._set_unknown_encoding_handler() return new_parser - at __pypy__.builtinify + at builtinify def ErrorString(errno): return XML_ErrorString(errno)[:200] - at __pypy__.builtinify + at builtinify def ParserCreate(encoding=None, namespace_separator=None, intern=None): if (not isinstance(encoding, str) and not encoding is None): Modified: pypy/trunk/lib_pypy/resource.py ============================================================================== --- pypy/trunk/lib_pypy/resource.py (original) +++ pypy/trunk/lib_pypy/resource.py Thu Dec 2 11:14:53 2010 @@ -1,4 +1,4 @@ -import sys, __pypy__ +import sys if sys.platform == 'win32': raise ImportError('resource module not available for win32') @@ -11,6 +11,10 @@ from errno import EINVAL, EPERM import _structseq +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + class error(Exception): pass @@ -77,7 +81,7 @@ ru_nvcsw = _structseq.structseqfield(14) ru_nivcsw = _structseq.structseqfield(15) - at __pypy__.builtinify + at builtinify def rlimit_check_bounds(rlim_cur, rlim_max): if rlim_cur > rlim_t_max: raise ValueError("%d does not fit into rlim_t" % rlim_cur) @@ -90,7 +94,7 @@ ("rlim_max", rlim_t), ) - at __pypy__.builtinify + at builtinify def getrusage(who): ru = _struct_rusage() ret = _getrusage(who, byref(ru)) @@ -118,7 +122,7 @@ ru.ru_nivcsw, )) - at __pypy__.builtinify + at builtinify def getrlimit(resource): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -130,7 +134,7 @@ raise error(errno) return (rlim.rlim_cur, rlim.rlim_max) - at __pypy__.builtinify + at builtinify def setrlimit(resource, rlim): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -147,7 +151,7 @@ else: raise error(errno) - at __pypy__.builtinify + at builtinify def getpagesize(): pagesize = 0 if _getpagesize: Modified: pypy/trunk/lib_pypy/syslog.py ============================================================================== --- pypy/trunk/lib_pypy/syslog.py (original) +++ pypy/trunk/lib_pypy/syslog.py Thu Dec 2 11:14:53 2010 @@ -5,7 +5,7 @@ syslog facility. """ -import sys, __pypy__ +import sys if sys.platform == 'win32': raise ImportError("No syslog on Windows") @@ -15,6 +15,10 @@ from ctypes_support import standard_c_lib as libc from ctypes import c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + # Real prototype is: # void syslog(int priority, const char *format, ...); # But we also need format ("%s") and one format argument (message) @@ -34,11 +38,11 @@ _setlogmask.argtypes = (c_int,) _setlogmask.restype = c_int - at __pypy__.builtinify + at builtinify def openlog(ident, option, facility): _openlog(ident, option, facility) - at __pypy__.builtinify + at builtinify def syslog(arg1, arg2=None): if arg2 is not None: priority, message = arg1, arg2 @@ -46,19 +50,19 @@ priority, message = LOG_INFO, arg1 _syslog(priority, "%s", message) - at __pypy__.builtinify + at builtinify def closelog(): _closelog() - at __pypy__.builtinify + at builtinify def setlogmask(mask): return _setlogmask(mask) - at __pypy__.builtinify + at builtinify def LOG_MASK(pri): return (1 << pri) - at __pypy__.builtinify + at builtinify def LOG_UPTO(pri): return (1 << (pri + 1)) - 1 From arigo at codespeak.net Thu Dec 2 11:51:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 2 Dec 2010 11:51:36 +0100 (CET) Subject: [pypy-svn] r79738 - pypy/trunk/pypy/rlib Message-ID: <20101202105136.42DF0282BD4@codespeak.net> Author: arigo Date: Thu Dec 2 11:51:34 2010 New Revision: 79738 Modified: pypy/trunk/pypy/rlib/jit.py Log: Fix comment. Modified: pypy/trunk/pypy/rlib/jit.py ============================================================================== --- pypy/trunk/pypy/rlib/jit.py (original) +++ pypy/trunk/pypy/rlib/jit.py Thu Dec 2 11:51:34 2010 @@ -156,7 +156,7 @@ def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1, arg3=-sys.maxint-1, arg4=-sys.maxint-1): - """When JITted, cause an extra operation DEBUG_MERGE_POINT to appear in + """When JITted, cause an extra operation JIT_DEBUG to appear in the graphs. Should not be left after debugging.""" keepalive_until_here(string) # otherwise the whole function call is removed jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)' From arigo at codespeak.net Thu Dec 2 11:58:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 2 Dec 2010 11:58:34 +0100 (CET) Subject: [pypy-svn] r79739 - pypy/branch/fast-forward/pypy/translator/goal Message-ID: <20101202105834.CC227282BE3@codespeak.net> Author: arigo Date: Thu Dec 2 11:58:33 2010 New Revision: 79739 Modified: pypy/branch/fast-forward/pypy/translator/goal/app_main.py Log: Fix the merge from trunk of the -E option. Modified: pypy/branch/fast-forward/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/goal/app_main.py (original) +++ pypy/branch/fast-forward/pypy/translator/goal/app_main.py Thu Dec 2 11:58:33 2010 @@ -205,7 +205,7 @@ break # found! return newpath -def setup_initial_paths(executable, nanos, readenv=True, **extra): +def setup_initial_paths(executable, nanos, ignore_environment=False, **extra): # a substituted os if we are translated global os os = nanos @@ -226,6 +226,7 @@ sys.executable = os.path.abspath(executable) newpath = get_library_path(executable) + readenv = not ignore_environment path = readenv and os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath @@ -271,7 +272,6 @@ options['warnoptions'] = [] print_sys_flags = False i = 0 - readenv = True while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -311,8 +311,6 @@ argv[i] = '-c' options["run_command"] = True break - elif arg == '-E': - readenv = False elif arg == '-u': options["unbuffered"] = True elif arg == '-O' or arg == '-OO': @@ -381,7 +379,7 @@ run_stdin, warnoptions, unbuffered, - readenv, + ignore_environment, cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 @@ -403,7 +401,8 @@ except: print >> sys.stderr, "'import site' failed" - pythonwarnings = os.getenv('PYTHONWARNINGS') + readenv = not ignore_environment + pythonwarnings = readenv and os.getenv('PYTHONWARNINGS') if pythonwarnings: warnoptions.extend(pythonwarnings.split(',')) if warnoptions: From fijal at codespeak.net Thu Dec 2 12:26:20 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 12:26:20 +0100 (CET) Subject: [pypy-svn] r79740 - in pypy/branch/out-of-line-guards/pypy/rpython: . lltypesystem test Message-ID: <20101202112620.1C4E4282B9D@codespeak.net> Author: fijal Date: Thu Dec 2 12:26:19 2010 New Revision: 79740 Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py pypy/branch/out-of-line-guards/pypy/rpython/rclass.py pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py Log: Fix Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py Thu Dec 2 12:26:19 2010 @@ -517,7 +517,8 @@ # for virtualizables; see rvirtualizable2.py if (op == 'setfield' and cname.value.startswith('inst_') and cname.value[len('inst_'):] in self.jit_invariant_fields): - llops.genop('jit_invariant_setfield', []) + llops.genop('jit_marker', [Constant('invariant_setfield', + lltype.Void)]) def new_instance(self, llops, classcallhop=None): """Build a new instance, without calling __init__.""" Modified: pypy/branch/out-of-line-guards/pypy/rpython/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/rclass.py Thu Dec 2 12:26:19 2010 @@ -170,7 +170,7 @@ self.immutable_fields = immutable_fields.value accessor = FieldListAccessor() hints['immutable_fields'] = accessor - if self.jit_invariant_fields: + if self.classdef.classdesc.lookup('_jit_invariant_fields_') is not None: hints = hints.copy() accessor = FieldListAccessor() hints['jit_invariant_fields'] = accessor Modified: pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py Thu Dec 2 12:26:19 2010 @@ -1069,8 +1069,12 @@ t, typer, graph = self.gengraph(f, []) block = graph.iterblocks().next() - assert 'jit_invariant_setfield' in [op.opname for op - in block.operations] + for op in block.operations: + if op.opname == 'jit_marker': + assert op.args[0].value == 'invariant_setfield' + break + else: + raise Exception("did not find jit invariant setfield marker") class TestOOtype(BaseTestRclass, OORtypeMixin): def test__del__(self): From afa at codespeak.net Thu Dec 2 14:03:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 2 Dec 2010 14:03:15 +0100 (CET) Subject: [pypy-svn] r79741 - pypy/branch/fast-forward/pypy/translator/goal/test2 Message-ID: <20101202130315.44937282B9D@codespeak.net> Author: afa Date: Thu Dec 2 14:03:14 2010 New Revision: 79741 Modified: pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py Log: dont print the sys.flags both in the test (the second one is wrong btw: it comes from the host CPython) Modified: pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py Thu Dec 2 14:03:14 2010 @@ -409,7 +409,7 @@ ("bytes_warning", "-b", "1"), ) for flag, opt, value in flags: - cmd = "%s --print-sys-flags -c 'import sys; print sys.flags'" + cmd = "%s --print-sys-flags -c pass" data = self.run(cmd % (opt,), expect_prompt=opt == "-i") assert "%s=%s" % (flag, value) in data From fijal at codespeak.net Thu Dec 2 14:08:52 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 14:08:52 +0100 (CET) Subject: [pypy-svn] r79742 - in pypy/branch/out-of-line-guards/pypy/rpython: lltypesystem test Message-ID: <20101202130852.B45C2282B9D@codespeak.net> Author: fijal Date: Thu Dec 2 14:08:51 2010 New Revision: 79742 Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py Log: Revert last checkin changes, jit_invariant_field sounds better Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py Thu Dec 2 14:08:51 2010 @@ -517,8 +517,7 @@ # for virtualizables; see rvirtualizable2.py if (op == 'setfield' and cname.value.startswith('inst_') and cname.value[len('inst_'):] in self.jit_invariant_fields): - llops.genop('jit_marker', [Constant('invariant_setfield', - lltype.Void)]) + llops.genop('jit_invariant_setfield', []) def new_instance(self, llops, classcallhop=None): """Build a new instance, without calling __init__.""" Modified: pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py Thu Dec 2 14:08:51 2010 @@ -1070,11 +1070,10 @@ t, typer, graph = self.gengraph(f, []) block = graph.iterblocks().next() for op in block.operations: - if op.opname == 'jit_marker': - assert op.args[0].value == 'invariant_setfield' + if op.opname == 'jit_invariant_setfield': break else: - raise Exception("did not find jit invariant setfield marker") + raise Exception("did not find jit invariant setfield") class TestOOtype(BaseTestRclass, OORtypeMixin): def test__del__(self): From fijal at codespeak.net Thu Dec 2 14:09:40 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 14:09:40 +0100 (CET) Subject: [pypy-svn] r79743 - pypy/branch/out-of-line-guards/pypy/jit/metainterp Message-ID: <20101202130940.30B52282BD4@codespeak.net> Author: fijal Date: Thu Dec 2 14:09:38 2010 New Revision: 79743 Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/virtualref.py Log: make VRefInfo a newstyle class Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/virtualref.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/virtualref.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/virtualref.py Thu Dec 2 14:09:38 2010 @@ -4,7 +4,7 @@ from pypy.jit.codewriter import heaptracker -class VirtualRefInfo: +class VirtualRefInfo(object): def __init__(self, warmrunnerdesc): self.warmrunnerdesc = warmrunnerdesc From antocuni at codespeak.net Thu Dec 2 14:11:04 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 2 Dec 2010 14:11:04 +0100 (CET) Subject: [pypy-svn] r79744 - pypy/build/bot2/pypybuildbot Message-ID: <20101202131104.DDB49282BDD@codespeak.net> Author: antocuni Date: Thu Dec 2 14:11:03 2010 New Revision: 79744 Modified: pypy/build/bot2/pypybuildbot/master.py Log: enable macosx64 tests nightly Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu Dec 2 14:11:03 2010 @@ -199,6 +199,7 @@ Nightly("nightly-0-45", [ JITBENCH, # on tannit -- nothing else there during first round! MACOSX32, # on minime + JITMACOSX64, # on mvt's machine ], hour=0, minute=45), Nightly("nightly-4-00", [ # rule: what we pick here on tannit should take at most 8 cores From arigo at codespeak.net Thu Dec 2 14:12:41 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 2 Dec 2010 14:12:41 +0100 (CET) Subject: [pypy-svn] r79745 - pypy/trunk/pypy/rpython/memory/gc/test Message-ID: <20101202131241.60BDB282BE0@codespeak.net> Author: arigo Date: Thu Dec 2 14:12:38 2010 New Revision: 79745 Modified: pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py Log: Add a test, failing with the minimark gc. Modified: pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/test/test_direct.py Thu Dec 2 14:12:38 2010 @@ -342,6 +342,15 @@ self.gc.collect() assert hash == self.gc.identityhash(self.stackroots[-1]) self.stackroots.pop() + # (7) the same, but the objects are dying young + for i in range(10): + self.gc.collect() + p = self.malloc(VAR, i) + self.stackroots.append(p) + hash1 = self.gc.identityhash(p) + hash2 = self.gc.identityhash(p) + assert hash1 == hash2 + self.stackroots.pop() def test_memory_alignment(self): A1 = lltype.GcArray(lltype.Char) From fijal at codespeak.net Thu Dec 2 14:12:55 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 14:12:55 +0100 (CET) Subject: [pypy-svn] r79746 - in pypy/branch/out-of-line-guards/pypy/jit/metainterp: . test Message-ID: <20101202131255.83E70282BE3@codespeak.net> Author: fijal Date: Thu Dec 2 14:12:52 2010 New Revision: 79746 Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py Log: Rewrite jit_invariant_setfield. Right now it simply vanishes Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmspot.py Thu Dec 2 14:12:52 2010 @@ -2,9 +2,11 @@ from pypy.jit.metainterp.warmspot import ll_meta_interp from pypy.jit.metainterp.warmspot import get_stats from pypy.rlib.jit import JitDriver, OPTIMIZER_FULL, OPTIMIZER_SIMPLE -from pypy.rlib.jit import unroll_safe +from pypy.rlib.jit import unroll_safe, dont_look_inside from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.history import BoxInt +from pypy.jit.codewriter.support import annotate +from pypy.jit.metainterp.warmspot import WarmRunnerDesc from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin @@ -359,7 +361,6 @@ self.check_loops({'int_sub': 1, 'int_gt': 1, 'guard_true': 1, 'jump': 1}) - class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLtypeCPU type_system = 'lltype' @@ -371,8 +372,6 @@ class TestWarmspotDirect(object): def setup_class(cls): from pypy.jit.metainterp.typesystem import llhelper - from pypy.jit.codewriter.support import annotate - from pypy.jit.metainterp.warmspot import WarmRunnerDesc from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.rpython.lltypesystem import lltype, llmemory exc_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) @@ -437,6 +436,7 @@ rtyper = annotate(f, [0]) FakeCPU.rtyper = rtyper + cls.FakeCPU = FakeCPU translator = rtyper.annotator.translator translator.config.translation.gc = 'hybrid' cls.desc = WarmRunnerDesc(translator, CPUClass=FakeCPU) @@ -454,3 +454,37 @@ assert lle[0] == self.exc_vtable else: py.test.fail("DID NOT RAISE") + + def test_rewrite_invariant_setfield(self): + from pypy.jit.metainterp.warmspot import find_jit_invariant_setfield + + class A(object): + _jit_invariant_fields_ = ['x'] + + @dont_look_inside + def g(a): + a.x = 3 + + a = A() + + driver = JitDriver(reds = ['i', 'a'], greens = []) + + def f(): + i = 0 + a = A() + g(a) + while i < 10: + driver.can_enter_jit(i=i, a=a) + driver.jit_merge_point(i=i, a=a) + i += 1 + return i + + g(a) + return a.x + + rtyper = annotate(f, []) + self.FakeCPU.rtyper = rtyper + translator = rtyper.annotator.translator + translator.config.translation.gc = 'hybrid' + desc = WarmRunnerDesc(translator, CPUClass=self.FakeCPU) + assert not find_jit_invariant_setfield(translator.graphs) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py Thu Dec 2 14:12:52 2010 @@ -114,6 +114,16 @@ results.append((graph, block, i)) return results +def find_jit_invariant_setfield(graphs): + results = [] + for graph in graphs: + for block in graph.iterblocks(): + for i in range(len(block.operations)): + op = block.operations[i] + if op.opname == 'jit_invariant_setfield': + results.append((graph, block, i)) + return results + def find_can_enter_jit(graphs): return _find_jit_marker(graphs, 'can_enter_jit') @@ -181,6 +191,7 @@ self.rewrite_can_enter_jits() self.rewrite_set_param() self.rewrite_force_virtual(vrefinfo) + self.rewrite_jit_invariant_setfield() self.add_finish() self.metainterp_sd.finish_setup(self.codewriter, optimizer=optimizer) @@ -802,6 +813,11 @@ op.opname = 'direct_call' op.args[:3] = [closures[key]] + def rewrite_jit_invariant_setfield(self): + graphs = self.translator.graphs + for graph, block, i in find_jit_invariant_setfield(graphs): + del block.operations[i] # for now + def rewrite_force_virtual(self, vrefinfo): if self.cpu.ts.name != 'lltype': py.test.skip("rewrite_force_virtual: port it to ootype") From afa at codespeak.net Thu Dec 2 14:17:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 2 Dec 2010 14:17:19 +0100 (CET) Subject: [pypy-svn] r79747 - in pypy/branch/fast-forward: lib_pypy pypy/doc/config pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/metainterp/optimizeopt pypy/module/_minimal_curses pypy/module/array/benchmark pypy/module/array/test pypy/module/test_lib_pypy/ctypes_tests pypy/rlib pypy/rlib/test pypy/rpython Message-ID: <20101202131719.283C1282B90@codespeak.net> Author: afa Date: Thu Dec 2 14:17:16 2010 New Revision: 79747 Modified: pypy/branch/fast-forward/lib_pypy/_locale.py pypy/branch/fast-forward/lib_pypy/_marshal.py pypy/branch/fast-forward/lib_pypy/_minimal_curses.py pypy/branch/fast-forward/lib_pypy/cPickle.py pypy/branch/fast-forward/lib_pypy/cmath.py pypy/branch/fast-forward/lib_pypy/grp.py pypy/branch/fast-forward/lib_pypy/itertools.py pypy/branch/fast-forward/lib_pypy/msvcrt.py pypy/branch/fast-forward/lib_pypy/pwd.py pypy/branch/fast-forward/lib_pypy/pyexpat.py pypy/branch/fast-forward/lib_pypy/resource.py pypy/branch/fast-forward/lib_pypy/syslog.py pypy/branch/fast-forward/pypy/doc/config/objspace.usemodules.array.txt (props changed) pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/gc.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-forward/pypy/jit/backend/x86/regloc.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/fast-forward/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/branch/fast-forward/pypy/module/_minimal_curses/__init__.py pypy/branch/fast-forward/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/fast-forward/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py pypy/branch/fast-forward/pypy/rlib/jit.py pypy/branch/fast-forward/pypy/rlib/rerased.py (props changed) pypy/branch/fast-forward/pypy/rlib/test/test_rerased.py (props changed) pypy/branch/fast-forward/pypy/rpython/extfunc.py Log: Merge from trunk: -r79701:79741 Modified: pypy/branch/fast-forward/lib_pypy/_locale.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_locale.py (original) +++ pypy/branch/fast-forward/lib_pypy/_locale.py Thu Dec 2 14:17:16 2010 @@ -11,7 +11,8 @@ # load the platform-specific cache made by running locale.ctc.py from ctypes_config_cache._locale_cache import * -import __pypy__ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f # Ubuntu Gusty i386 structure @@ -160,7 +161,7 @@ ul = ''.join(ul) string.letters = ul - at __pypy__.builtinify + at builtinify def setlocale(category, locale=None): "(integer,string=None) -> string. Activates/queries locale processing." if locale: @@ -185,7 +186,7 @@ groups.append(0) return groups - at __pypy__.builtinify + at builtinify def localeconv(): "() -> dict. Returns numeric and monetary locale-specific parameters." @@ -219,7 +220,7 @@ } return result - at __pypy__.builtinify + at builtinify def strcoll(s1, s2): "string,string -> int. Compares two strings according to the locale." @@ -238,7 +239,7 @@ # Collate the strings. return _wcscoll(s1, s2) - at __pypy__.builtinify + at builtinify def strxfrm(s): "string -> string. Returns a string that behaves for cmp locale-aware." @@ -252,7 +253,7 @@ _strxfrm(buf, s, n2) return buf.value - at __pypy__.builtinify + at builtinify def getdefaultlocale(): # TODO: Port code from CPython for Windows and Mac OS raise NotImplementedError() @@ -274,31 +275,31 @@ raise ValueError("unsupported langinfo constant") if HAS_LIBINTL: - @__pypy__.builtinify + @builtinify def gettext(msg): """gettext(msg) -> string Return translation of msg.""" return _gettext(msg) - @__pypy__.builtinify + @builtinify def dgettext(domain, msg): """dgettext(domain, msg) -> string Return translation of msg in domain.""" return _dgettext(domain, msg) - @__pypy__.builtinify + @builtinify def dcgettext(domain, msg, category): """dcgettext(domain, msg, category) -> string Return translation of msg in domain and category.""" return _dcgettext(domain, msg, category) - @__pypy__.builtinify + @builtinify def textdomain(domain): """textdomain(domain) -> string Set the C library's textdomain to domain, returning the new domain.""" return _textdomain(domain) - @__pypy__.builtinify + @builtinify def bindtextdomain(domain, dir): """bindtextdomain(domain, dir) -> string Bind the C library's domain to dir.""" @@ -309,7 +310,7 @@ return dirname if HAS_BIND_TEXTDOMAIN_CODESET: - @__pypy__.builtinify + @builtinify def bind_textdomain_codeset(domain, codeset): """bind_textdomain_codeset(domain, codeset) -> string Bind the C library's domain to codeset.""" Modified: pypy/branch/fast-forward/lib_pypy/_marshal.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_marshal.py (original) +++ pypy/branch/fast-forward/lib_pypy/_marshal.py Thu Dec 2 14:17:16 2010 @@ -3,9 +3,13 @@ This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, transport the file to a Sun, and read it back there). Details of the format may change between Python versions. """ -import types, __pypy__ +import types from _codecs import utf_8_decode, utf_8_encode +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + TYPE_NULL = '0' TYPE_NONE = 'N' TYPE_FALSE = 'F' @@ -645,18 +649,18 @@ version = 1 - at __pypy__.builtinify + at builtinify def dump(x, f, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format m = _Marshaller(f.write) m.dump(x) - at __pypy__.builtinify + at builtinify def load(f): um = _Unmarshaller(f.read) return um.load() - at __pypy__.builtinify + at builtinify def dumps(x, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format buffer = [] @@ -664,7 +668,7 @@ m.dump(x) return ''.join(buffer) - at __pypy__.builtinify + at builtinify def loads(s): um = _FastUnmarshaller(s) return um.load() Modified: pypy/branch/fast-forward/lib_pypy/_minimal_curses.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_minimal_curses.py (original) +++ pypy/branch/fast-forward/lib_pypy/_minimal_curses.py Thu Dec 2 14:17:16 2010 @@ -35,23 +35,24 @@ # ____________________________________________________________ -import __pypy__ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f - at __pypy__.builtinify + at builtinify def setupterm(termstr, fd): err = ctypes.c_int(0) result = clib.setupterm(termstr, fd, ctypes.byref(err)) if result == ERR: raise error("setupterm() failed (err=%d)" % err.value) - at __pypy__.builtinify + at builtinify def tigetstr(cap): result = clib.tigetstr(cap) if ctypes.cast(result, ctypes.c_void_p).value == ERR: return None return ctypes.cast(result, ctypes.c_char_p).value - at __pypy__.builtinify + at builtinify def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) if result is None: Modified: pypy/branch/fast-forward/lib_pypy/cPickle.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/cPickle.py (original) +++ pypy/branch/fast-forward/lib_pypy/cPickle.py Thu Dec 2 14:17:16 2010 @@ -4,7 +4,10 @@ from pickle import * from pickle import __doc__, __version__, format_version, compatible_formats -import __pypy__ + +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + BadPickleGet = KeyError UnpickleableError = PicklingError @@ -32,11 +35,11 @@ def getvalue(self): return self.__f and self.__f.getvalue() - at __pypy__.builtinify + at builtinify def dump(obj, file, protocol=None): Pickler(file, protocol).dump(obj) - at __pypy__.builtinify + at builtinify def dumps(obj, protocol=None): file = StringIO() Pickler(file, protocol).dump(obj) Modified: pypy/branch/fast-forward/lib_pypy/cmath.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/cmath.py (original) +++ pypy/branch/fast-forward/lib_pypy/cmath.py Thu Dec 2 14:17:16 2010 @@ -5,9 +5,12 @@ # much code borrowed from mathmodule.c -import math, __pypy__ +import math from math import e, pi +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + # constants _one = complex(1., 0.) @@ -32,14 +35,13 @@ return complex(real, imag) - - at __pypy__.builtinify + at builtinify def phase(x): x = _to_complex(x) return math.atan2(x.imag, x.real) - at __pypy__.builtinify + at builtinify def polar(x): x = _to_complex(x) phi = math.atan2(x.imag, x.real) @@ -47,12 +49,12 @@ return r, phi - at __pypy__.builtinify + at builtinify def rect(r, phi): return complex(r * math.cos(phi), r * math.sin(phi)) - at __pypy__.builtinify + at builtinify def acos(x): """acos(x) @@ -62,7 +64,7 @@ return -(_prodi(log((x+(_i*sqrt((_one-(x*x)))))))) - at __pypy__.builtinify + at builtinify def acosh(x): """acosh(x) @@ -73,7 +75,7 @@ return z+z - at __pypy__.builtinify + at builtinify def asin(x): """asin(x) @@ -86,7 +88,7 @@ return -(_prodi(log((sqrt_1_minus_x_sq+_prodi(x))))) - at __pypy__.builtinify + at builtinify def asinh(x): """asinh(x) @@ -97,7 +99,7 @@ return z+z - at __pypy__.builtinify + at builtinify def atan(x): """atan(x) @@ -107,7 +109,7 @@ return _halfi*log(((_i+x)/(_i-x))) - at __pypy__.builtinify + at builtinify def atanh(x): """atanh(x) @@ -117,7 +119,7 @@ return _half*log((_one+x)/(_one-x)) - at __pypy__.builtinify + at builtinify def cos(x): """cos(x) @@ -129,7 +131,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def cosh(x): """cosh(x) @@ -141,7 +143,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def exp(x): """exp(x) @@ -154,7 +156,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def log(x, base=None): """log(x) @@ -169,7 +171,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def log10(x): """log10(x) @@ -182,7 +184,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def sin(x): """sin(x) @@ -194,7 +196,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def sinh(x): """sinh(x) @@ -206,7 +208,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def sqrt(x): """sqrt(x) @@ -232,7 +234,7 @@ _sqrt_half = sqrt(_half) - at __pypy__.builtinify + at builtinify def tan(x): """tan(x) @@ -253,7 +255,7 @@ return complex(real, imag) - at __pypy__.builtinify + at builtinify def tanh(x): """tanh(x) Modified: pypy/branch/fast-forward/lib_pypy/grp.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/grp.py (original) +++ pypy/branch/fast-forward/lib_pypy/grp.py Thu Dec 2 14:17:16 2010 @@ -2,13 +2,17 @@ """ This module provides ctypes version of cpython's grp module """ -import sys, __pypy__ +import sys if sys.platform == 'win32': raise ImportError("No grp module on Windows") from ctypes import Structure, c_char_p, c_int, POINTER from ctypes_support import standard_c_lib as libc +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + gid_t = c_int class GroupStruct(Structure): @@ -64,7 +68,7 @@ return Group(res.contents.gr_name, res.contents.gr_passwd, res.contents.gr_gid, mem) - at __pypy__.builtinify + at builtinify def getgrgid(gid): res = libc.getgrgid(gid) if not res: @@ -72,7 +76,7 @@ raise KeyError(gid) return _group_from_gstruct(res) - at __pypy__.builtinify + at builtinify def getgrnam(name): if not isinstance(name, str): raise TypeError("expected string") @@ -81,7 +85,7 @@ raise KeyError(name) return _group_from_gstruct(res) - at __pypy__.builtinify + at builtinify def getgrall(): libc.setgrent() lst = [] Modified: pypy/branch/fast-forward/lib_pypy/itertools.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/itertools.py (original) +++ pypy/branch/fast-forward/lib_pypy/itertools.py Thu Dec 2 14:17:16 2010 @@ -27,7 +27,8 @@ 'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap', 'takewhile', 'tee'] -import __pypy__ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f class chain(object): @@ -634,7 +635,7 @@ return self - at __pypy__.builtinify + at builtinify def tee(iterable, n=2): """Return n independent iterators from a single iterable. Note : once tee() has made a split, the original iterable Modified: pypy/branch/fast-forward/lib_pypy/msvcrt.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/msvcrt.py (original) +++ pypy/branch/fast-forward/lib_pypy/msvcrt.py Thu Dec 2 14:17:16 2010 @@ -11,13 +11,16 @@ from ctypes_support import standard_c_lib as _c from ctypes_support import get_errno import errno -import __pypy__ try: open_osfhandle = _c._open_osfhandle except AttributeError: # we are not on windows raise ImportError +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int] open_osfhandle.restype = ctypes.c_int @@ -35,7 +38,7 @@ _locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] _locking.restype = ctypes.c_int - at __pypy__.builtinify + at builtinify def locking(fd, mode, nbytes): '''lock or unlock a number of bytes in a file.''' rv = _locking(fd, mode, nbytes) Modified: pypy/branch/fast-forward/lib_pypy/pwd.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/pwd.py (original) +++ pypy/branch/fast-forward/lib_pypy/pwd.py Thu Dec 2 14:17:16 2010 @@ -10,13 +10,17 @@ exception is raised if the entry asked for cannot be found. """ -import sys, __pypy__ +import sys if sys.platform == 'win32': raise ImportError("No pwd module on Windows") from ctypes_support import standard_c_lib as libc from ctypes import Structure, POINTER, c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + uid_t = c_int gid_t = c_int @@ -79,12 +83,12 @@ _endpwent.argtypes = None _endpwent.restype = None - at __pypy__.builtinify + at builtinify def mkpwent(pw): pw = pw.contents return struct_passwd(pw) - at __pypy__.builtinify + at builtinify def getpwuid(uid): """ getpwuid(uid) -> (pw_name,pw_passwd,pw_uid, @@ -97,7 +101,7 @@ raise KeyError("getpwuid(): uid not found: %s" % uid) return mkpwent(pw) - at __pypy__.builtinify + at builtinify def getpwnam(name): """ getpwnam(name) -> (pw_name,pw_passwd,pw_uid, @@ -112,7 +116,7 @@ raise KeyError("getpwname(): name not found: %s" % name) return mkpwent(pw) - at __pypy__.builtinify + at builtinify def getpwall(): """ getpwall() -> list_of_entries Modified: pypy/branch/fast-forward/lib_pypy/pyexpat.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/pyexpat.py (original) +++ pypy/branch/fast-forward/lib_pypy/pyexpat.py Thu Dec 2 14:17:16 2010 @@ -2,11 +2,14 @@ import ctypes import ctypes.util from ctypes import c_char_p, c_int, c_void_p, POINTER, c_char, c_wchar_p -import sys, __pypy__ +import sys # load the platform-specific cache made by running pyexpat.ctc.py from ctypes_config_cache._pyexpat_cache import * +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + lib = ctypes.CDLL(ctypes.util.find_library('expat')) @@ -425,11 +428,11 @@ new_parser._set_unknown_encoding_handler() return new_parser - at __pypy__.builtinify + at builtinify def ErrorString(errno): return XML_ErrorString(errno)[:200] - at __pypy__.builtinify + at builtinify def ParserCreate(encoding=None, namespace_separator=None, intern=None): if (not isinstance(encoding, str) and not encoding is None): Modified: pypy/branch/fast-forward/lib_pypy/resource.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/resource.py (original) +++ pypy/branch/fast-forward/lib_pypy/resource.py Thu Dec 2 14:17:16 2010 @@ -1,4 +1,4 @@ -import sys, __pypy__ +import sys if sys.platform == 'win32': raise ImportError('resource module not available for win32') @@ -11,6 +11,10 @@ from errno import EINVAL, EPERM import _structseq +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + class error(Exception): pass @@ -77,7 +81,7 @@ ru_nvcsw = _structseq.structseqfield(14) ru_nivcsw = _structseq.structseqfield(15) - at __pypy__.builtinify + at builtinify def rlimit_check_bounds(rlim_cur, rlim_max): if rlim_cur > rlim_t_max: raise ValueError("%d does not fit into rlim_t" % rlim_cur) @@ -90,7 +94,7 @@ ("rlim_max", rlim_t), ) - at __pypy__.builtinify + at builtinify def getrusage(who): ru = _struct_rusage() ret = _getrusage(who, byref(ru)) @@ -118,7 +122,7 @@ ru.ru_nivcsw, )) - at __pypy__.builtinify + at builtinify def getrlimit(resource): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -130,7 +134,7 @@ raise error(errno) return (rlim.rlim_cur, rlim.rlim_max) - at __pypy__.builtinify + at builtinify def setrlimit(resource, rlim): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -147,7 +151,7 @@ else: raise error(errno) - at __pypy__.builtinify + at builtinify def getpagesize(): pagesize = 0 if _getpagesize: Modified: pypy/branch/fast-forward/lib_pypy/syslog.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/syslog.py (original) +++ pypy/branch/fast-forward/lib_pypy/syslog.py Thu Dec 2 14:17:16 2010 @@ -5,7 +5,7 @@ syslog facility. """ -import sys, __pypy__ +import sys if sys.platform == 'win32': raise ImportError("No syslog on Windows") @@ -15,6 +15,10 @@ from ctypes_support import standard_c_lib as libc from ctypes import c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + # Real prototype is: # void syslog(int priority, const char *format, ...); # But we also need format ("%s") and one format argument (message) @@ -34,11 +38,11 @@ _setlogmask.argtypes = (c_int,) _setlogmask.restype = c_int - at __pypy__.builtinify + at builtinify def openlog(ident, option, facility): _openlog(ident, option, facility) - at __pypy__.builtinify + at builtinify def syslog(arg1, arg2=None): if arg2 is not None: priority, message = arg1, arg2 @@ -46,19 +50,19 @@ priority, message = LOG_INFO, arg1 _syslog(priority, "%s", message) - at __pypy__.builtinify + at builtinify def closelog(): _closelog() - at __pypy__.builtinify + at builtinify def setlogmask(mask): return _setlogmask(mask) - at __pypy__.builtinify + at builtinify def LOG_MASK(pri): return (1 << pri) - at __pypy__.builtinify + at builtinify def LOG_UPTO(pri): return (1 << (pri + 1)) - 1 Modified: pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llgraph/llimpl.py Thu Dec 2 14:17:16 2010 @@ -583,7 +583,12 @@ def op_debug_merge_point(self, _, value, recdepth): from pypy.jit.metainterp.warmspot import get_stats loc = ConstPtr(value)._get_str() - get_stats().add_merge_point_location(loc) + try: + stats = get_stats() + except AttributeError: + pass + else: + stats.add_merge_point_location(loc) def op_guard_true(self, _, value): if not value: Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py Thu Dec 2 14:17:16 2010 @@ -43,6 +43,22 @@ self.total_mallocs -= (stop - start) self._add_free_block(start, stop) + def open_malloc(self, minsize): + """Allocate at least minsize bytes. Returns (start, stop).""" + result = self._allocate_block(minsize) + (start, stop) = result + self.total_mallocs += stop - start + return result + + def open_free(self, middle, stop): + """Used for freeing the end of an open-allocated block of memory.""" + if stop - middle >= self.min_fragment: + self.total_mallocs -= (stop - middle) + self._add_free_block(middle, stop) + return True + else: + return False # too small to record + def _allocate_large_block(self, minsize): # Compute 'size' from 'minsize': it must be rounded up to # 'large_alloc_size'. Additionally, we use the following line @@ -140,6 +156,40 @@ self._allocated = None +class MachineDataBlockWrapper(object): + def __init__(self, asmmemmgr, allblocks): + self.asmmemmgr = asmmemmgr + self.allblocks = allblocks + self.rawstart = 0 + self.rawposition = 0 + self.rawstop = 0 + + def done(self): + if self.rawstart != 0: + if self.asmmemmgr.open_free(self.rawposition, self.rawstop): + self.rawstop = self.rawposition + self.allblocks.append((self.rawstart, self.rawstop)) + self.rawstart = 0 + self.rawposition = 0 + self.rawstop = 0 + + def _allocate_next_block(self, minsize): + self.done() + self.rawstart, self.rawstop = self.asmmemmgr.open_malloc(minsize) + self.rawposition = self.rawstart + + def malloc_aligned(self, size, alignment): + p = self.rawposition + p = (p + alignment - 1) & (-alignment) + if p + size > self.rawstop: + self._allocate_next_block(size + alignment - 1) + p = self.rawposition + p = (p + alignment - 1) & (-alignment) + assert p + size <= self.rawstop + self.rawposition = p + size + return p + + class BlockBuilderMixin(object): _mixin_ = True # A base class to generate assembler. It is equivalent to just a list @@ -156,7 +206,6 @@ SUBBLOCK_PTR.TO.become(SUBBLOCK) gcroot_markers = None - gcroot_markers_total_size = 0 def __init__(self, translated=None): if translated is None: @@ -224,11 +273,8 @@ self.copy_to_raw_memory(rawstart) if self.gcroot_markers is not None: assert gcrootmap is not None - gcrootmap.add_raw_gcroot_markers(asmmemmgr, - allblocks, - self.gcroot_markers, - self.gcroot_markers_total_size, - rawstart) + for pos, mark in self.gcroot_markers: + gcrootmap.put(rawstart + pos, mark) return rawstart def _become_a_plain_block_builder(self): @@ -247,4 +293,3 @@ if self.gcroot_markers is None: self.gcroot_markers = [] self.gcroot_markers.append((self.get_relative_pos(), mark)) - self.gcroot_markers_total_size += len(mark) Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/gc.py Thu Dec 2 14:17:16 2010 @@ -222,7 +222,7 @@ LOC_EBP_MINUS = 3 GCMAP_ARRAY = rffi.CArray(lltype.Signed) - CALLSHAPE_ARRAY = rffi.CArray(rffi.UCHAR) + CALLSHAPE_ARRAY_PTR = rffi.CArrayPtr(rffi.UCHAR) def __init__(self): # '_gcmap' is an array of length '_gcmap_maxlength' of addresses. @@ -264,8 +264,7 @@ self._gcmap_sorted = True return sorted - @rgc.no_collect - def _put(self, retaddr, callshapeaddr): + def put(self, retaddr, callshapeaddr): """'retaddr' is the address just after the CALL. 'callshapeaddr' is the address of the raw 'shape' marker. Both addresses are actually integers here.""" @@ -308,37 +307,6 @@ lltype.free(oldgcmap, flavor='raw', track_allocation=False) return j - def add_raw_gcroot_markers(self, asmmemmgr, allblocks, - markers, total_size, rawstart): - """The interface is a bit custom, but this routine writes the - shapes of gcroots (for the GC to use) into raw memory.""" - # xxx so far, we never try to share them. But right now - # the amount of potential sharing would not be too large. - dst = 1 - stop = 0 - for relpos, shape in markers: - # - if dst + len(shape) > stop: - # No more space in the previous raw block, - # allocate a raw block of memory big enough to fit - # as many of the remaining 'shapes' as possible - start, stop = asmmemmgr.malloc(len(shape), total_size) - # add the raw block to 'compiled_loop_token.asmmemmgr_blocks' - allblocks.append((start, stop)) - dst = start - # - # add the entry 'pos_after_call -> dst' to the table - self._put(rawstart + relpos, dst) - # Copy 'shape' into the raw memory, reversing the order - # of the bytes. Similar to compress_callshape() in - # trackgcroot.py. - total_size -= len(shape) - src = len(shape) - 1 - while src >= 0: - rffi.cast(rffi.CCHARP, dst)[0] = shape[src] - dst += 1 - src -= 1 - @rgc.no_collect def freeing_block(self, start, stop): # if [start:stop] is a raw block of assembler, then look up the @@ -409,6 +377,16 @@ assert reg_index > 0 shape.append(chr(self.LOC_REG | (reg_index << 2))) + def compress_callshape(self, shape, datablockwrapper): + # Similar to compress_callshape() in trackgcroot.py. + # Returns an address to raw memory (as an integer). + length = len(shape) + rawaddr = datablockwrapper.malloc_aligned(length, 1) + p = rffi.cast(self.CALLSHAPE_ARRAY_PTR, rawaddr) + for i in range(length): + p[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) + return rawaddr + class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py Thu Dec 2 14:17:16 2010 @@ -1,5 +1,6 @@ import random from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager +from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from pypy.rpython.lltypesystem import lltype, rffi @@ -151,14 +152,10 @@ prev_total = new_total def test_insert_gcroot_marker(self): + puts = [] class FakeGcRootMap: - def add_raw_gcroot_markers(self, asmmemmgr, allblocks, markers, - total_size, rawstart): - self.asmmemmgr = asmmemmgr - self.allblocks = allblocks - self.markers = markers - self.total_size = total_size - self.rawstart = rawstart + def put(self, retaddr, mark): + puts.append((retaddr, mark)) # mc = BlockBuilderMixin() mc.writechar('X') @@ -181,10 +178,8 @@ assert p[4] == 'Z' assert p[5] == 'z' assert allblocks == [(rawstart, rawstart + 6)] - assert gcrootmap.markers == [(2, ['a', 'b', 'c', 'd']), - (4, ['e', 'f', 'g'])] - assert gcrootmap.total_size == 4 + 3 - assert gcrootmap.rawstart == rawstart + assert puts == [(rawstart + 2, ['a', 'b', 'c', 'd']), + (rawstart + 4, ['e', 'f', 'g'])] def test_blockbuildermixin(translated=True): @@ -215,3 +210,41 @@ def test_blockbuildermixin2(): test_blockbuildermixin(translated=False) + +def test_machinedatablock(): + ops = [] + class FakeMemMgr: + _addr = 1597 + def open_malloc(self, minsize): + result = (self._addr, self._addr + 100) + ops.append(('malloc', minsize) + result) + self._addr += 200 + return result + def open_free(self, frm, to): + ops.append(('free', frm, to)) + return to - frm >= 8 + # + allblocks = [] + md = MachineDataBlockWrapper(FakeMemMgr(), allblocks) + p = md.malloc_aligned(26, 16) + assert p == 1600 + assert ops == [('malloc', 26 + 15, 1597, 1697)] + del ops[:] + # + p = md.malloc_aligned(26, 16) + assert p == 1632 + p = md.malloc_aligned(26, 16) + assert p == 1664 + assert allblocks == [] + assert ops == [] + # + p = md.malloc_aligned(27, 16) + assert p == 1808 + assert allblocks == [(1597, 1697)] + assert ops == [('free', 1690, 1697), + ('malloc', 27 + 15, 1797, 1897)] + del ops[:] + # + md.done() + assert allblocks == [(1597, 1697), (1797, 1835)] + assert ops == [('free', 1835, 1897)] Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_gc.py Thu Dec 2 14:17:16 2010 @@ -91,11 +91,27 @@ assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, 4, 8, 12, 16]) + def test_compress_callshape(self): + class FakeDataBlockWrapper: + def malloc_aligned(self, size, alignment): + assert alignment == 1 # here + assert size == 4 + return rffi.cast(lltype.Signed, p) + datablockwrapper = FakeDataBlockWrapper() + p = lltype.malloc(rffi.CArray(lltype.Char), 4, immortal=True) + gcrootmap = GcRootMap_asmgcc() + shape = ['a', 'b', 'c', 'd'] + gcrootmap.compress_callshape(shape, datablockwrapper) + assert p[0] == 'd' + assert p[1] == 'c' + assert p[2] == 'b' + assert p[3] == 'a' + def test_put_basic(self): gcrootmap = GcRootMap_asmgcc() retaddr = 1234567890 shapeaddr = 51627384 - gcrootmap._put(retaddr, shapeaddr) + gcrootmap.put(retaddr, shapeaddr) assert gcrootmap._gcmap[0] == retaddr assert gcrootmap._gcmap[1] == shapeaddr p = rffi.cast(rffi.LONGP, gcrootmap.gcmapstart()) @@ -109,7 +125,7 @@ for i in range(700): shapeaddr = i * 100 + 1 retaddr = 123456789 + i - gcrootmap._put(retaddr, shapeaddr) + gcrootmap.put(retaddr, shapeaddr) for i in range(700): assert gcrootmap._gcmap[i*2+0] == 123456789 + i assert gcrootmap._gcmap[i*2+1] == i * 100 + 1 @@ -126,7 +142,7 @@ for i in range(700): shapeaddr = i * 100 # 0 if i == 0 retaddr = 123456789 + i - gcrootmap._put(retaddr, shapeaddr) + gcrootmap.put(retaddr, shapeaddr) if shapeaddr != 0: expected.append((retaddr, shapeaddr)) # at the first resize, the 0 should be removed @@ -142,72 +158,11 @@ # check that we can again insert 350 entries without a resize oldgcmap = gcrootmap._gcmap for i in range(0, 699, 2): - gcrootmap._put(515151 + i + repeat, 626262 + i) + gcrootmap.put(515151 + i + repeat, 626262 + i) expected.append((515151 + i + repeat, 626262 + i)) assert gcrootmap._gcmap == oldgcmap check() - def test_add_raw_gcroot_markers_maxalloc(self): - class FakeAsmMemMgr: - def malloc(self, minsize, maxsize): - assert minsize == 4 - assert maxsize == 7 - return (prawstart, prawstart + 8) - put = [] - def fakeput(a, b): - put.append((a, b)) - gcrootmap = GcRootMap_asmgcc() - gcrootmap._put = fakeput - memmgr = FakeAsmMemMgr() - allblocks = [] - p = lltype.malloc(rffi.CArray(lltype.Char), 7, immortal=True) - prawstart = rffi.cast(lltype.Signed, p) - gcrootmap.add_raw_gcroot_markers(memmgr, allblocks, - [(2, ['a', 'b', 'c', 'd']), - (4, ['e', 'f', 'g'])], - 4 + 3, 1200000) - assert allblocks == [(prawstart, prawstart + 8)] - assert ''.join([p[i] for i in range(7)]) == 'dcbagfe' - assert put == [(1200002, prawstart), - (1200004, prawstart + 4)] - - def test_add_raw_gcroot_markers_minalloc(self): - class FakeAsmMemMgr: - callnum = 0 - def malloc(self, minsize, maxsize): - self.callnum += 1 - if self.callnum == 1: - assert minsize == 4 - assert maxsize == 7 - return (prawstart, prawstart + 6) - elif self.callnum == 2: - assert minsize == 3 - assert maxsize == 3 - return (qrawstart, qrawstart + 5) - else: - raise AssertionError - put = [] - def fakeput(a, b): - put.append((a, b)) - gcrootmap = GcRootMap_asmgcc() - gcrootmap._put = fakeput - memmgr = FakeAsmMemMgr() - allblocks = [] - p = lltype.malloc(rffi.CArray(lltype.Char), 6, immortal=True) - prawstart = rffi.cast(lltype.Signed, p) - q = lltype.malloc(rffi.CArray(lltype.Char), 5, immortal=True) - qrawstart = rffi.cast(lltype.Signed, q) - gcrootmap.add_raw_gcroot_markers(memmgr, allblocks, - [(2, ['a', 'b', 'c', 'd']), - (4, ['e', 'f', 'g'])], - 4 + 3, 1200000) - assert allblocks == [(prawstart, prawstart + 6), - (qrawstart, qrawstart + 5)] - assert ''.join([p[i] for i in range(4)]) == 'dcba' - assert ''.join([q[i] for i in range(3)]) == 'gfe' - assert put == [(1200002, prawstart), - (1200004, qrawstart)] - def test_freeing_block(self): from pypy.jit.backend.llsupport import gc class Asmgcroot: Modified: pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/test/runner_test.py Thu Dec 2 14:17:16 2010 @@ -1203,6 +1203,13 @@ yield nan_and_infinity, rop.FLOAT_GT, operator.gt, all_cases_binary yield nan_and_infinity, rop.FLOAT_GE, operator.ge, all_cases_binary + def test_noops(self): + c_box = self.alloc_string("hi there").constbox() + c_nest = ConstInt(0) + self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest], 'void') + self.execute_operation(rop.JIT_DEBUG, [c_box, c_nest, c_nest, + c_nest, c_nest], 'void') + class LLtypeBackendTest(BaseBackendTest): Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py Thu Dec 2 14:17:16 2010 @@ -1,5 +1,6 @@ import sys, os from pypy.jit.backend.llsupport import symbolic +from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT, LoopToken) @@ -55,7 +56,6 @@ DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) class Assembler386(object): - _float_constants = None _regalloc = None _output_loop_log = None @@ -83,6 +83,7 @@ self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') self.fail_boxes_count = 0 self._current_depths_cache = (0, 0) + self.datablockwrapper = None self.teardown() def leave_jitted_hook(self): @@ -125,10 +126,14 @@ self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') - def setup(self): + def setup(self, looptoken): assert self.memcpy_addr != 0, "setup_once() not called?" self.pending_guard_tokens = [] self.mc = codebuf.MachineCodeBlockWrapper() + if self.datablockwrapper is None: + allblocks = self.get_asmmemmgr_blocks(looptoken) + self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, + allblocks) def teardown(self): self.pending_guard_tokens = None @@ -145,13 +150,9 @@ debug_stop('jit-backend-counts') def _build_float_constants(self): - # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment - addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw', - track_allocation=False) - if not we_are_translated(): - self._keepalive_malloced_float_consts = addr - float_constants = rffi.cast(lltype.Signed, addr) - float_constants = (float_constants + 15) & ~15 # align to 16 bytes + datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, []) + float_constants = datablockwrapper.malloc_aligned(32, alignment=16) + datablockwrapper.done() addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' # 0x8000000000000000 @@ -203,13 +204,18 @@ _x86_arglocs _x86_debug_checksum ''' + # XXX this function is too longish and contains some code + # duplication with assemble_bridge(). Also, we should think + # about not storing on 'self' attributes that will live only + # for the duration of compiling one loop or a one bridge. + clt = CompiledLoopToken(self.cpu, looptoken.number) looptoken.compiled_loop_token = clt if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.setup() + self.setup(looptoken) self.currently_compiling_loop = looptoken funcname = self._find_debug_merge_point(operations) if log: @@ -235,7 +241,7 @@ self.write_pending_failure_recoveries() fullsize = self.mc.get_relative_pos() # - rawstart = self.materialize(looptoken) + rawstart = self.materialize_loop(looptoken) debug_print("Loop #%d (%s) has address %x to %x" % ( looptoken.number, funcname, rawstart + self.looppos, @@ -268,7 +274,7 @@ "was already compiled!") return - self.setup() + self.setup(original_loop_token) funcname = self._find_debug_merge_point(operations) if log: self._register_counter() @@ -289,7 +295,7 @@ self.write_pending_failure_recoveries() fullsize = self.mc.get_relative_pos() # - rawstart = self.materialize(original_loop_token) + rawstart = self.materialize_loop(original_loop_token) debug_print("Bridge out of guard %d (%s) has address %x to %x" % (descr_number, funcname, rawstart, rawstart + codeendpos)) @@ -328,12 +334,17 @@ p = rffi.cast(rffi.INTP, addr) p[0] = rffi.cast(rffi.INT, relative_target) - def materialize(self, looptoken): + def get_asmmemmgr_blocks(self, looptoken): clt = looptoken.compiled_loop_token if clt.asmmemmgr_blocks is None: clt.asmmemmgr_blocks = [] - return self.mc.materialize(self.cpu.asmmemmgr, - clt.asmmemmgr_blocks, + return clt.asmmemmgr_blocks + + def materialize_loop(self, looptoken): + self.datablockwrapper.done() # finish using cpu.asmmemmgr + self.datablockwrapper = None + allblocks = self.get_asmmemmgr_blocks(looptoken) + return self.mc.materialize(self.cpu.asmmemmgr, allblocks, self.cpu.gc_ll_descr.gcrootmap) def _find_debug_merge_point(self, operations): Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py Thu Dec 2 14:17:16 2010 @@ -60,32 +60,6 @@ r15: 5, } -class FloatConstants(object): - BASE_CONSTANT_SIZE = 1000 - - def __init__(self): - self.cur_array_free = 0 - self.const_id = 0 - - def _get_new_array(self): - n = self.BASE_CONSTANT_SIZE - # known to leak - self.cur_array = lltype.malloc(rffi.CArray(lltype.Float), n, # YYY leak - flavor='raw', track_allocation=False) - self.cur_array_free = n - _get_new_array._dont_inline_ = True - - def record_float(self, floatval): - if self.cur_array_free == 0: - self._get_new_array() - arr = self.cur_array - n = self.cur_array_free - 1 - arr[n] = floatval - self.cur_array_free = n - self.const_id += 1 - return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) - - class X86XMMRegisterManager(RegisterManager): box_types = [FLOAT] @@ -93,20 +67,11 @@ # we never need lower byte I hope save_around_call_regs = all_regs - def __init__(self, longevity, frame_manager=None, assembler=None): - RegisterManager.__init__(self, longevity, frame_manager=frame_manager, - assembler=assembler) - if assembler is None: - self.float_constants = FloatConstants() - else: - if assembler._float_constants is None: - assembler._float_constants = FloatConstants() - self.float_constants = assembler._float_constants - def convert_to_imm(self, c): - const_id, adr = self.float_constants.record_float(c.getfloat()) - return ConstFloatLoc(adr, const_id) - + adr = self.assembler.datablockwrapper.malloc_aligned(8, 8) + rffi.cast(rffi.CArrayPtr(rffi.DOUBLE), adr)[0] = c.getfloat() + return ConstFloatLoc(adr) + def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately @@ -1108,7 +1073,8 @@ if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert reg in self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX gcrootmap.add_callee_save_reg(shape, self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX[reg]) - return shape + return gcrootmap.compress_callshape(shape, + self.assembler.datablockwrapper) def consider_force_token(self, op): loc = self.rm.force_allocate_reg(op.result) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/regloc.py Thu Dec 2 14:17:16 2010 @@ -177,24 +177,15 @@ class ConstFloatLoc(AssemblerLocation): # XXX: We have to use this class instead of just AddressLoc because - # AddressLoc is "untyped" and also we to have need some sort of unique - # identifier that we can use in _getregkey (for jump.py) - + # we want a width of 8 (... I think. Check this!) _immutable_ = True - width = 8 - def __init__(self, address, const_id): + def __init__(self, address): self.value = address - self.const_id = const_id def __repr__(self): - return '' % (self.value, self.const_id) - - def _getregkey(self): - # XXX: 1000 is kind of magic: We just don't want to be confused - # with any registers - return 1000 + self.const_id + return '' % (self.value,) def location_code(self): return 'j' Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_gc_integration.py Thu Dec 2 14:17:16 2010 @@ -33,6 +33,10 @@ def add_callee_save_reg(self, shape, reg_index): index_to_name = { 1: 'ebx', 2: 'esi', 3: 'edi' } shape.append(index_to_name[reg_index]) + def compress_callshape(self, shape, datablockwrapper): + assert datablockwrapper == 'fakedatablockwrapper' + assert shape[0] == 'shape' + return ['compressed'] + shape[1:] class MockGcDescr(GcCache): def get_funcptr_for_new(self): @@ -57,6 +61,7 @@ cpu = CPU(None, None) cpu.setup_once() regalloc = RegAlloc(MockAssembler(cpu, MockGcDescr(False))) + regalloc.assembler.datablockwrapper = 'fakedatablockwrapper' boxes = [BoxPtr() for i in range(len(X86RegisterManager.all_regs))] longevity = {} for box in boxes: @@ -79,7 +84,7 @@ assert len(regalloc.assembler.movs) == 3 # mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) - assert mark[0] == 'shape' + assert mark[0] == 'compressed' base = -WORD * FRAME_FIXED_SIZE expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_regalloc.py Thu Dec 2 14:17:16 2010 @@ -9,7 +9,7 @@ from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ - FloatConstants, is_comparison_or_ovf_op + is_comparison_or_ovf_op from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -520,16 +520,6 @@ self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] - def test_float_overflow_const_list(self): - ops = ['[f0]'] - BASE_CONSTANT_SIZE = FloatConstants.BASE_CONSTANT_SIZE - for i in range(BASE_CONSTANT_SIZE * 2): - ops.append('f%d = float_add(f%d, 3.5)' % (i + 1, i)) - ops.append('finish(f%d)' % (BASE_CONSTANT_SIZE * 2)) - ops = "\n".join(ops) - self.interpret(ops, [0.1]) - assert abs(self.getfloat(0) - (BASE_CONSTANT_SIZE * 2) * 3.5 - 0.1) < 0.00001 - def test_lt_const(self): ops = ''' [f0] Modified: pypy/branch/fast-forward/pypy/module/_minimal_curses/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_minimal_curses/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/_minimal_curses/__init__.py Thu Dec 2 14:17:16 2010 @@ -4,7 +4,7 @@ try: import _minimal_curses as _curses # when running on top of pypy-c except ImportError: - import py; py.test.skip("no _curses module") # no _curses at all + raise ImportError("no _curses or _minimal_curses module") # no _curses at all from pypy.interpreter.mixedmodule import MixedModule from pypy.module._minimal_curses import fficurses Modified: pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py (original) +++ pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py Thu Dec 2 14:17:16 2010 @@ -11,21 +11,23 @@ py.test.skip("pypy white-box test") from _ctypes.function import CFuncPtr - guess = CFuncPtr._guess_argtypes + def guess(value): + cobj = CFuncPtr._conv_param(None, value, 0) + return type(cobj) - assert guess([13]) == [c_int] - assert guess([0]) == [c_int] - assert guess(['xca']) == [c_char_p] - assert guess([None]) == [c_void_p] - assert guess([c_int(3)]) == [c_int] - assert guess([u'xca']) == [c_wchar_p] + assert guess(13) == c_int + assert guess(0) == c_int + assert guess('xca') == c_char_p + assert guess(None) == c_void_p + assert guess(c_int(3)) == c_int + assert guess(u'xca') == c_wchar_p class Stuff: pass s = Stuff() s._as_parameter_ = None - assert guess([s]) == [c_void_p] + assert guess(s) == c_void_p def test_guess_unicode(): if not hasattr(sys, 'pypy_translation_info') and sys.platform != 'win32': Modified: pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py (original) +++ pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py Thu Dec 2 14:17:16 2010 @@ -99,7 +99,7 @@ def test_primitive(self): if not hasattr(sys, 'pypy_translation_info'): py.test.skip("pypy white-box test") - assert c_char_p("abc")._objects['0']._buffer[0] == "a" + assert c_char_p("abc")._objects._buffer[0] == "a" assert c_int(3)._objects is None def test_pointer_to_pointer(self): @@ -123,7 +123,7 @@ pass cf = CFUNCTYPE(c_int, c_int)(f) p1 = cast(cf, c_void_p) - assert p1._objects == {'1': cf, '0': {'0': cf}} + assert p1._objects == {id(cf): cf, '0': cf} def test_array_of_struct_with_pointer(self): class S(Structure): @@ -221,7 +221,7 @@ import gc; gc.collect() print 'x =', repr(x) assert x.value == 'hellohello' - assert x._objects.keys() == ['0'] + assert x._objects == 'hellohello' # class datum(Structure): _fields_ = [ Modified: pypy/branch/fast-forward/pypy/rlib/jit.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/jit.py (original) +++ pypy/branch/fast-forward/pypy/rlib/jit.py Thu Dec 2 14:17:16 2010 @@ -156,7 +156,7 @@ def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1, arg3=-sys.maxint-1, arg4=-sys.maxint-1): - """When JITted, cause an extra operation DEBUG_MERGE_POINT to appear in + """When JITted, cause an extra operation JIT_DEBUG to appear in the graphs. Should not be left after debugging.""" keepalive_until_here(string) # otherwise the whole function call is removed jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)' Modified: pypy/branch/fast-forward/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/extfunc.py (original) +++ pypy/branch/fast-forward/pypy/rpython/extfunc.py Thu Dec 2 14:17:16 2010 @@ -201,6 +201,11 @@ exec py.code.compile(""" from pypy.rlib.objectmodel import running_on_llinterp from pypy.rlib.debug import llinterpcall + from pypy.rlib.jit import dont_look_inside + # note: we say 'dont_look_inside' mostly because the + # JIT does not support 'running_on_llinterp', but in + # theory it is probably right to stop jitting anyway. + @dont_look_inside def ll_wrapper(%s): if running_on_llinterp: return llinterpcall(s_result, fakeimpl, %s) From arigo at codespeak.net Thu Dec 2 14:29:03 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 2 Dec 2010 14:29:03 +0100 (CET) Subject: [pypy-svn] r79748 - pypy/trunk/pypy/rpython/memory/gc Message-ID: <20101202132903.6F42F282B90@codespeak.net> Author: arigo Date: Thu Dec 2 14:29:01 2010 New Revision: 79748 Modified: pypy/trunk/pypy/rpython/memory/gc/minimark.py Log: Fix the bug shown in r79745. Modified: pypy/trunk/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/minimark.py Thu Dec 2 14:29:01 2010 @@ -652,8 +652,13 @@ # means recording that they have a smaller size, so that when # moved out of the nursery, they will consume less memory. # In particular, an array with GCFLAG_HAS_CARDS is never resized. + # Also, a nursery object with GCFLAG_HAS_SHADOW is not resized + # either, as this would potentially loose part of the memory in + # the already-allocated shadow. if not self.is_in_nursery(obj): return False + if self.header(obj).tid & GCFLAG_HAS_SHADOW: + return False # size_gc_header = self.gcheaderbuilder.size_gc_header typeid = self.get_type_id(obj) @@ -1423,12 +1428,21 @@ size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + size) - # initialize to an invalid tid *without* GCFLAG_VISITED, - # so that if the object dies before the next minor - # collection, the shadow will stay around but be collected - # by the next major collection. + # Initialize the shadow enough to be considered a + # valid gc object. If the original object stays + # alive at the next minor collection, it will anyway + # be copied over the shadow and overwrite the + # following fields. But if the object dies, then + # the shadow will stay around and only be freed at + # the next major collection, at which point we want + # it to look valid (but ready to be freed). shadow = shadowhdr + size_gc_header - self.header(shadow).tid = 0 + self.header(shadow).tid = self.header(obj).tid + typeid = self.get_type_id(obj) + if self.is_varsize(typeid): + lenofs = self.varsize_offset_to_length(typeid) + (shadow + lenofs).signed[0] = (obj + lenofs).signed[0] + # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.young_objects_shadows.setitem(obj, shadow) # From antocuni at codespeak.net Thu Dec 2 14:38:08 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 2 Dec 2010 14:38:08 +0100 (CET) Subject: [pypy-svn] r79749 - pypy/build/bot2/pypybuildbot Message-ID: <20101202133808.47E89282BE0@codespeak.net> Author: antocuni Date: Thu Dec 2 14:38:06 2010 New Revision: 79749 Modified: pypy/build/bot2/pypybuildbot/master.py Log: schedule jitmacosx64 at 4:00 instead of 0:45 Modified: pypy/build/bot2/pypybuildbot/master.py ============================================================================== --- pypy/build/bot2/pypybuildbot/master.py (original) +++ pypy/build/bot2/pypybuildbot/master.py Thu Dec 2 14:38:06 2010 @@ -199,7 +199,6 @@ Nightly("nightly-0-45", [ JITBENCH, # on tannit -- nothing else there during first round! MACOSX32, # on minime - JITMACOSX64, # on mvt's machine ], hour=0, minute=45), Nightly("nightly-4-00", [ # rule: what we pick here on tannit should take at most 8 cores @@ -210,6 +209,7 @@ OJITLINUX32, # on tannit32, uses 1 core APPLVLWIN32, # on bigboard STACKLESSAPPLVLFREEBSD64, # on headless + JITMACOSX64, # on mvt's machine ], hour=4, minute=0), Nightly("nightly-6-00", [ # there should be only JITLINUX32 that takes a bit longer than From antocuni at codespeak.net Thu Dec 2 15:06:28 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 2 Dec 2010 15:06:28 +0100 (CET) Subject: [pypy-svn] r79750 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20101202140628.DF215282B90@codespeak.net> Author: antocuni Date: Thu Dec 2 15:06:26 2010 New Revision: 79750 Added: pypy/trunk/lib-python/modified-2.5.2/test/seq_tests.py - copied, changed from r79749, pypy/trunk/lib-python/2.5.2/test/seq_tests.py Log: make this test working also on 64 bit Copied: pypy/trunk/lib-python/modified-2.5.2/test/seq_tests.py (from r79749, pypy/trunk/lib-python/2.5.2/test/seq_tests.py) ============================================================================== --- pypy/trunk/lib-python/2.5.2/test/seq_tests.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/seq_tests.py Thu Dec 2 15:06:26 2010 @@ -307,11 +307,19 @@ self.assertEqual(id(s), id(s*1)) def test_bigrepeat(self): + import sys + # we chose an N such as 2**16 * N does not fit into a cpu word + if sys.maxint == 2147483647: + # 32 bit system + N = 2**16 + else: + # 64 bit system + N = 2**48 x = self.type2test([0]) x *= 2**16 - self.assertRaises(MemoryError, x.__mul__, 2**16) + self.assertRaises(MemoryError, x.__mul__, N) if hasattr(x, '__imul__'): - self.assertRaises(MemoryError, x.__imul__, 2**16) + self.assertRaises(MemoryError, x.__imul__, N) def test_subscript(self): a = self.type2test([10, 11]) From cfbolz at codespeak.net Thu Dec 2 15:13:53 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 2 Dec 2010 15:13:53 +0100 (CET) Subject: [pypy-svn] r79751 - pypy/trunk/pypy/rpython/memory/gctransform Message-ID: <20101202141353.116D55080B@codespeak.net> Author: cfbolz Date: Thu Dec 2 15:13:51 2010 New Revision: 79751 Modified: pypy/trunk/pypy/rpython/memory/gctransform/boehm.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py pypy/trunk/pypy/rpython/memory/gctransform/refcounting.py pypy/trunk/pypy/rpython/memory/gctransform/support.py Log: issue586 testing Try to make the error messages of interp-level destructors more helpful by printing the name of the TYPE and the exception raised. Modified: pypy/trunk/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/boehm.py Thu Dec 2 15:13:51 2010 @@ -120,10 +120,11 @@ fptr = self.annotate_finalizer(d['ll_finalizer'], [llmemory.Address], lltype.Void) elif destrptr: EXC_INSTANCE_TYPE = self.translator.rtyper.exceptiondata.lltype_of_exception_value + typename = TYPE.__name__ def ll_finalizer(addr): exc_instance = llop.gc_fetch_exception(EXC_INSTANCE_TYPE) v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) llop.gc_restore_exception(lltype.Void, exc_instance) fptr = self.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) else: Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Thu Dec 2 15:13:51 2010 @@ -1204,9 +1204,10 @@ assert not type_contains_pyobjs(TYPE), "not implemented" if destrptr: + typename = TYPE.__name__ def ll_finalizer(addr): v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) fptr = self.transformer.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) Modified: pypy/trunk/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/refcounting.py Thu Dec 2 15:13:51 2010 @@ -227,7 +227,7 @@ # refcount is at zero, temporarily bump it to 1: gcheader.refcount = 1 destr_v = cast_pointer(DESTR_ARG, v) - ll_call_destructor(destrptr, destr_v) + ll_call_destructor(destrptr, destr_v, %r) refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: @@ -239,7 +239,7 @@ pop_alive(exc_instance) # XXX layering of exceptiontransform versus gcpolicy -""" % (body, TYPE._gckind) +""" % (TYPE.__name__, body, TYPE._gckind) else: call_del = None body = '\n'.join(_static_deallocator_body_for_type('v', TYPE)) Modified: pypy/trunk/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/support.py Thu Dec 2 15:13:51 2010 @@ -98,11 +98,15 @@ hop.exception_cannot_occur() return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const) -def ll_call_destructor(destrptr, destr_v): +def ll_call_destructor(destrptr, destr_v, typename): try: destrptr(destr_v) - except: + except Exception, e: try: - os.write(2, "a destructor raised an exception, ignoring it\n") + os.write(2, "a destructor of type ") + os.write(2, typename) + os.write(2, " raised an exception ") + os.write(2, str(e)) + os.write(2, " ignoring it\n") except: pass From cfbolz at codespeak.net Thu Dec 2 16:01:42 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 2 Dec 2010 16:01:42 +0100 (CET) Subject: [pypy-svn] r79752 - in pypy/trunk/pypy: annotation annotation/test rpython rpython/lltypesystem rpython/ootypesystem Message-ID: <20101202150142.00166282BD4@codespeak.net> Author: cfbolz Date: Thu Dec 2 16:01:39 2010 New Revision: 79752 Modified: pypy/trunk/pypy/annotation/annrpython.py pypy/trunk/pypy/annotation/bookkeeper.py pypy/trunk/pypy/annotation/description.py pypy/trunk/pypy/annotation/model.py pypy/trunk/pypy/annotation/specialize.py pypy/trunk/pypy/annotation/test/test_annrpython.py pypy/trunk/pypy/rpython/lltypesystem/rbuiltin.py pypy/trunk/pypy/rpython/lltypesystem/rclass.py pypy/trunk/pypy/rpython/lltypesystem/rdict.py pypy/trunk/pypy/rpython/lltypesystem/rpbc.py pypy/trunk/pypy/rpython/ootypesystem/rpbc.py pypy/trunk/pypy/rpython/rpbc.py Log: a small improvement in the annotator: use a real set instead of a dict for descriptions. not a big thing, but makes a few lines nicer. Modified: pypy/trunk/pypy/annotation/annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/annrpython.py (original) +++ pypy/trunk/pypy/annotation/annrpython.py Thu Dec 2 16:01:39 2010 @@ -145,7 +145,7 @@ classdef.add_source_for_attribute(attr, classdef.classdesc) self.bookkeeper assert isinstance(s_result, annmodel.SomePBC) - olddesc = s_result.descriptions.iterkeys().next() + olddesc = s_result.any_description() desc = olddesc.bind_self(classdef) args = self.bookkeeper.build_args("simple_call", args_s[:]) desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc], Modified: pypy/trunk/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/trunk/pypy/annotation/bookkeeper.py (original) +++ pypy/trunk/pypy/annotation/bookkeeper.py Thu Dec 2 16:01:39 2010 @@ -262,7 +262,7 @@ args_s, s_result) def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result): - descs = s_callable.descriptions.keys() + descs = list(s_callable.descriptions) if not descs: return family = descs[0].getcallfamily() @@ -590,7 +590,7 @@ assert s_attr.is_constant() attr = s_attr.const - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue @@ -633,7 +633,7 @@ """Analyse a call to a SomePBC() with the given args (list of annotations). """ - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue first = descs[0] Modified: pypy/trunk/pypy/annotation/description.py ============================================================================== --- pypy/trunk/pypy/annotation/description.py (original) +++ pypy/trunk/pypy/annotation/description.py Thu Dec 2 16:01:39 2010 @@ -672,7 +672,7 @@ if isinstance(s_init, SomePBC): assert len(s_init.descriptions) == 1, ( "unexpected dynamic __init__?") - initfuncdesc = s_init.descriptions.keys()[0] + initfuncdesc, = s_init.descriptions if isinstance(initfuncdesc, FunctionDesc): initmethdesc = bookkeeper.getmethoddesc(initfuncdesc, classdef, @@ -800,8 +800,8 @@ desc.selfclassdef, desc.name, commonflags) - del descs[desc] - descs[newdesc] = None + descs.remove(desc) + descs.add(newdesc) # --- case 1 --- groups = {} @@ -816,7 +816,7 @@ for desc2 in group: cdef2 = desc2.selfclassdef if cdef1 is not cdef2 and cdef1.issubclass(cdef2): - del descs[desc1] + descs.remove(desc1) break simplify_desc_set = staticmethod(simplify_desc_set) Modified: pypy/trunk/pypy/annotation/model.py ============================================================================== --- pypy/trunk/pypy/annotation/model.py (original) +++ pypy/trunk/pypy/annotation/model.py Thu Dec 2 16:01:39 2010 @@ -356,8 +356,8 @@ immutable = True def __init__(self, descriptions, can_be_None=False, subset_of=None): - # descriptions is a set of Desc instances. - descriptions = dict.fromkeys(descriptions) + # descriptions is a set of Desc instances + descriptions = set(descriptions) self.descriptions = descriptions self.can_be_None = can_be_None self.subset_of = subset_of @@ -379,6 +379,9 @@ if desc.pyobj is not None: self.const = desc.pyobj + def any_description(self): + return iter(self.descriptions).next() + def getKind(self): "Return the common Desc class of all descriptions in this PBC." kinds = {} Modified: pypy/trunk/pypy/annotation/specialize.py ============================================================================== --- pypy/trunk/pypy/annotation/specialize.py (original) +++ pypy/trunk/pypy/annotation/specialize.py Thu Dec 2 16:01:39 2010 @@ -345,7 +345,8 @@ key.append(s.const) elif isinstance(s, SomePBC) and len(s.descriptions) == 1: # for test_specialize_arg_bound_method - key.append(s.descriptions.keys()[0]) + desc, = s.descriptions + key.append(desc) else: raise Exception("specialize:arg(%d): argument not constant: %r" % (i, s)) Modified: pypy/trunk/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/trunk/pypy/annotation/test/test_annrpython.py (original) +++ pypy/trunk/pypy/annotation/test/test_annrpython.py Thu Dec 2 16:01:39 2010 @@ -1010,7 +1010,7 @@ bookkeeper = a.bookkeeper def getmdesc(bmeth): - return bookkeeper.immutablevalue(bmeth).descriptions.keys()[0] + return bookkeeper.immutablevalue(bmeth).any_description() mdescA_m = getmdesc(A().m) mdescC_m = getmdesc(C().m) @@ -2862,7 +2862,7 @@ assert s.items[0].flags == {'access_directly': True} assert isinstance(s.items[1], annmodel.SomePBC) assert len(s.items[1].descriptions) == 1 - assert s.items[1].descriptions.keys()[0].flags == {'access_directly': + assert s.items[1].any_description().flags == {'access_directly': True} assert isinstance(s.items[2], annmodel.SomeInstance) assert s.items[2].flags == {'access_directly': True} Modified: pypy/trunk/pypy/rpython/lltypesystem/rbuiltin.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rbuiltin.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rbuiltin.py Thu Dec 2 16:01:39 2010 @@ -42,7 +42,7 @@ return hop.genop('cast_pointer', [v_inst], # v_type implicit in r_result resulttype = hop.r_result.lowleveltype) - classdef = s_class.descriptions.keys()[0].getuniqueclassdef() + classdef = s_class.any_description().getuniqueclassdef() return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops) def rtype_builtin_hasattr(hop): Modified: pypy/trunk/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rclass.py Thu Dec 2 16:01:39 2010 @@ -392,7 +392,7 @@ source_classdef = source_desc.getclassdef(None) source_repr = getinstancerepr(self.rtyper, source_classdef) assert len(s_func.descriptions) == 1 - funcdesc = s_func.descriptions.keys()[0] + funcdesc, = s_func.descriptions graph = funcdesc.getuniquegraph() FUNCTYPE = FuncType([Ptr(source_repr.object_type)], Void) destrptr = functionptr(FUNCTYPE, graph.name, Modified: pypy/trunk/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rdict.py Thu Dec 2 16:01:39 2010 @@ -581,7 +581,7 @@ def ll_dict_lookup_clean(d, hash): # a simplified version of ll_dict_lookup() which assumes that the # key is new, and the dictionary doesn't contain deleted entries. - # It only find the next free slot for the given hash. + # It only finds the next free slot for the given hash. entries = d.entries mask = len(entries) - 1 i = hash & mask Modified: pypy/trunk/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rpbc.py Thu Dec 2 16:01:39 2010 @@ -127,7 +127,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(self.rtyper, self.callfamily) assert len(uniquerows) == 1 @@ -166,7 +166,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) def convert_desc(self, funcdesc): @@ -230,7 +230,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness Modified: pypy/trunk/pypy/rpython/ootypesystem/rpbc.py ============================================================================== --- pypy/trunk/pypy/rpython/ootypesystem/rpbc.py (original) +++ pypy/trunk/pypy/rpython/ootypesystem/rpbc.py Thu Dec 2 16:01:39 2010 @@ -49,7 +49,7 @@ return hop.genop('runtimenew', [v_class], resulttype=resulttype) def getlowleveltype(self): - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) # if any of the classdefs get the lowleveltype ootype.Class, # we can only pick ootype.Class for us too. Otherwise META. for classdesc in classdescs: @@ -70,7 +70,7 @@ class MethodImplementations(object): def __init__(self, rtyper, methdescs): - samplemdesc = methdescs.iterkeys().next() + samplemdesc = iter(methdescs).next() concretetable, uniquerows = get_concrete_calltable(rtyper, samplemdesc.funcdesc.getcallfamily()) self.row_mapping = {} @@ -117,7 +117,7 @@ concretetable = None # set by _setup_repr_final def _setup_repr_final(self): - sampledesc = self.s_pbc.descriptions.iterkeys().next() + sampledesc = self.s_pbc.any_description() self.concretetable, _ = get_concrete_calltable(self.rtyper, sampledesc.funcdesc.getcallfamily()) Modified: pypy/trunk/pypy/rpython/rpbc.py ============================================================================== --- pypy/trunk/pypy/rpython/rpbc.py (original) +++ pypy/trunk/pypy/rpython/rpbc.py Thu Dec 2 16:01:39 2010 @@ -15,11 +15,10 @@ from pypy.rpython import callparse - def small_cand(rtyper, s_pbc): if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets and \ hasattr(rtyper.type_system.rpbc, 'SmallFunctionSetPBCRepr'): - callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(rtyper, callfamily) if len(uniquerows) == 1 and (not s_pbc.subset_of or small_cand(rtyper, s_pbc.subset_of)): return True @@ -31,7 +30,7 @@ return none_frozen_pbc_repr kind = self.getKind() if issubclass(kind, description.FunctionDesc): - sample = self.descriptions.keys()[0] + sample = self.any_description() callfamily = sample.querycallfamily() if callfamily and callfamily.total_calltable_size > 0: if sample.overridden: @@ -181,7 +180,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None: # a single function self.lowleveltype = Void @@ -207,7 +206,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) ## def function_signatures(self): @@ -322,7 +321,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness @@ -368,7 +367,7 @@ return robject.pyobj_repr def getFrozenPBCRepr(rtyper, s_pbc): - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) assert len(descs) >= 1 if len(descs) == 1 and not s_pbc.can_be_None: return SingleFrozenPBCRepr(descs[0]) @@ -530,7 +529,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper - self.funcdesc = s_pbc.descriptions.keys()[0].funcdesc + self.funcdesc = s_pbc.any_description().funcdesc # a hack to force the underlying function to show up in call_families # (generally not needed, as normalizecalls() should ensure this, @@ -662,7 +661,7 @@ and the ClassRepr of the class which stores this attribute in its vtable. """ - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) access = classdescs[0].queryattrfamily(attrname) for classdesc in classdescs[1:]: access1 = classdesc.queryattrfamily(attrname) @@ -819,7 +818,7 @@ if s_pbc.isNone(): raise TyperError("unsupported: variable of type " "bound-method-object or None") - mdescs = s_pbc.descriptions.keys() + mdescs = list(s_pbc.descriptions) methodname = mdescs[0].name classdef = mdescs[0].selfclassdef flags = mdescs[0].flags From fijal at codespeak.net Thu Dec 2 16:19:36 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 16:19:36 +0100 (CET) Subject: [pypy-svn] r79753 - in pypy/branch/out-of-line-guards/pypy/rpython: . ootypesystem Message-ID: <20101202151936.E49C8282B9D@codespeak.net> Author: fijal Date: Thu Dec 2 16:19:33 2010 New Revision: 79753 Modified: pypy/branch/out-of-line-guards/pypy/rpython/ootypesystem/rclass.py pypy/branch/out-of-line-guards/pypy/rpython/rvirtualizable2.py Log: fix virtualizables Modified: pypy/branch/out-of-line-guards/pypy/rpython/ootypesystem/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/ootypesystem/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/ootypesystem/rclass.py Thu Dec 2 16:19:33 2010 @@ -468,17 +468,17 @@ v_attr = inputconst(ootype.Void, mangled) r_value = self.allfields[mangled] self.lowleveltype._check_field(mangled) - self.hook_access_field(v_inst, v_attr, llops, flags) + self.hook_access_field('getfield', v_inst, v_attr, llops, flags) return llops.genop('oogetfield', [v_inst, v_attr], resulttype = r_value) def setfield(self, vinst, attr, vvalue, llops, flags={}): mangled_name = mangle(attr, self.rtyper.getconfig()) cname = inputconst(ootype.Void, mangled_name) - self.hook_access_field(vinst, cname, llops, flags) + self.hook_access_field('setfield', vinst, cname, llops, flags) llops.genop('oosetfield', [vinst, cname, vvalue]) - def hook_access_field(self, vinst, cname, llops, flags): + def hook_access_field(self, opname, vinst, cname, llops, flags): pass # for virtualizables; see rvirtualizable2.py def rtype_is_true(self, hop): Modified: pypy/branch/out-of-line-guards/pypy/rpython/rvirtualizable2.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/rvirtualizable2.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/rvirtualizable2.py Thu Dec 2 16:19:33 2010 @@ -36,7 +36,8 @@ self._super()._setup_repr(hints = hints) c_vfields = self.classdef.classdesc.classdict['_virtualizable2_'] self.my_redirected_fields = self._parse_field_list(c_vfields.value, - self.accessor) + self.accessor, + '_virtualizable2_') else: self._super()._setup_repr() # ootype needs my_redirected_fields even for subclass. lltype does From cfbolz at codespeak.net Thu Dec 2 17:05:59 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 2 Dec 2010 17:05:59 +0100 (CET) Subject: [pypy-svn] r79754 - pypy/trunk/pypy/rlib Message-ID: <20101202160559.645DD282B9D@codespeak.net> Author: cfbolz Date: Thu Dec 2 17:05:55 2010 New Revision: 79754 Modified: pypy/trunk/pypy/rlib/rerased.py Log: fix another place that used descriptions Modified: pypy/trunk/pypy/rlib/rerased.py ============================================================================== --- pypy/trunk/pypy/rlib/rerased.py (original) +++ pypy/trunk/pypy/rlib/rerased.py Thu Dec 2 17:05:55 2010 @@ -91,7 +91,7 @@ return annmodel.SomeInteger() assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() return annmodel.SomeInstance(clsdef) def specialize_call(self, hop): From arigo at codespeak.net Thu Dec 2 17:17:39 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 2 Dec 2010 17:17:39 +0100 (CET) Subject: [pypy-svn] r79755 - pypy/trunk/lib_pypy/_ctypes Message-ID: <20101202161739.77DB7282BE7@codespeak.net> Author: arigo Date: Thu Dec 2 17:17:37 2010 New Revision: 79755 Modified: pypy/trunk/lib_pypy/_ctypes/array.py pypy/trunk/lib_pypy/_ctypes/pointer.py pypy/trunk/lib_pypy/_ctypes/primitive.py Log: Remove the __new__ introduced to handle re-calling __init__() explicitly. Fix this case with just a hack. The issue is that the __new__ method is also called from various places in the code, and we don't want to create (and autofree only at the next gc collection) unneeded objects. Doing so is enough to make "pypy-c translate.py" run out of memory on 32-bit. Modified: pypy/trunk/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/array.py (original) +++ pypy/trunk/lib_pypy/_ctypes/array.py Thu Dec 2 17:17:37 2010 @@ -158,12 +158,9 @@ __metaclass__ = ArrayMeta _ffiargshape = 'P' - def __new__(cls, *args): - self = _CData.__new__(cls, *args) - self._buffer = self._ffiarray(self._length_, autofree=True) - return self - def __init__(self, *args): + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(self._length_, autofree=True) for i, arg in enumerate(args): self[i] = arg Modified: pypy/trunk/lib_pypy/_ctypes/pointer.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/pointer.py (original) +++ pypy/trunk/lib_pypy/_ctypes/pointer.py Thu Dec 2 17:17:37 2010 @@ -56,7 +56,8 @@ def set_type(self, TP): ffiarray = _rawffi.Array('P') def __init__(self, value=None): - self._buffer = ffiarray(1, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = ffiarray(1, autofree=True) if value is not None: self.contents = value self._ffiarray = ffiarray Modified: pypy/trunk/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/primitive.py (original) +++ pypy/trunk/lib_pypy/_ctypes/primitive.py Thu Dec 2 17:17:37 2010 @@ -288,12 +288,9 @@ __metaclass__ = SimpleType _type_ = 'i' - def __new__(cls, *args, **kwds): - self = _CData.__new__(cls, *args, **kwds) - self._buffer = self._ffiarray(1, autofree=True) - return self - def __init__(self, value=DEFAULT_VALUE): + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(1, autofree=True) if value is not DEFAULT_VALUE: self.value = value From cfbolz at codespeak.net Thu Dec 2 17:23:54 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Thu, 2 Dec 2010 17:23:54 +0100 (CET) Subject: [pypy-svn] r79756 - pypy/trunk/pypy/rlib Message-ID: <20101202162354.07615282BEB@codespeak.net> Author: cfbolz Date: Thu Dec 2 17:23:53 2010 New Revision: 79756 Modified: pypy/trunk/pypy/rlib/rerased.py Log: grmpf Modified: pypy/trunk/pypy/rlib/rerased.py ============================================================================== --- pypy/trunk/pypy/rlib/rerased.py (original) +++ pypy/trunk/pypy/rlib/rerased.py Thu Dec 2 17:23:53 2010 @@ -108,7 +108,7 @@ def compute_result_annotation(self, s_obj, s_type): assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() s_item = annmodel.SomeInstance(clsdef) return self.bookkeeper.newlist(s_item) From arigo at codespeak.net Thu Dec 2 18:29:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 2 Dec 2010 18:29:46 +0100 (CET) Subject: [pypy-svn] r79757 - pypy/trunk/lib_pypy/_ctypes Message-ID: <20101202172946.D67DF36C22E@codespeak.net> Author: arigo Date: Thu Dec 2 18:29:45 2010 New Revision: 79757 Modified: pypy/trunk/lib_pypy/_ctypes/basics.py Log: * adapt the __repr__ so that we have a clue that it's actually a CArgObject. * all objects with __eq__ should also have __ne__. Modified: pypy/trunk/lib_pypy/_ctypes/basics.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/basics.py (original) +++ pypy/trunk/lib_pypy/_ctypes/basics.py Thu Dec 2 18:29:45 2010 @@ -86,11 +86,14 @@ self._buffer = None def __repr__(self): - return repr(self._obj) + return '' % (self._obj,) def __eq__(self, other): return self._obj == other + def __ne__(self, other): + return self._obj != other + class _CData(object): """ The most basic object for all ctypes types """ From david at codespeak.net Thu Dec 2 18:40:32 2010 From: david at codespeak.net (david at codespeak.net) Date: Thu, 2 Dec 2010 18:40:32 +0100 (CET) Subject: [pypy-svn] r79758 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101202174032.1EBC936C22E@codespeak.net> Author: david Date: Thu Dec 2 18:40:30 2010 New Revision: 79758 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: Reduce the number of instructions used for guards Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Thu Dec 2 18:40:30 2010 @@ -1,7 +1,7 @@ from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import locations from pypy.jit.backend.arm import registers as r -from pypy.jit.backend.arm.arch import WORD, FUNC_ALIGN +from pypy.jit.backend.arm.arch import WORD, FUNC_ALIGN, PC_OFFSET from pypy.jit.backend.arm.codebuilder import ARMv7Builder, ARMv7InMemoryBuilder from pypy.jit.backend.arm.regalloc import ARMRegisterManager, ARMFrameManager from pypy.jit.backend.llsupport.regalloc import compute_vars_longevity, TempBox @@ -243,9 +243,10 @@ n = self.cpu.get_fail_descr_number(descr) self.encode32(mem, j+1, n) - self.mc.ensure_can_fit(self.mc.size_of_gen_load_int) - self.mc.gen_load_int(r.lr.value, memaddr, cond=fcond) # use lr to pass an argument + self.mc.ensure_can_fit(4*WORD) + self.mc.LDR_ri(r.lr.value, r.pc.value, imm=WORD) self.mc.B(self._exit_code_addr) + self.mc.write32(memaddr) return memaddr Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Thu Dec 2 18:40:30 2010 @@ -181,15 +181,15 @@ _mixin_ = True - guard_size = ARMv7Builder.size_of_gen_load_int + 6*WORD + guard_size = 10*WORD def _emit_guard(self, op, regalloc, fcond, save_exc=False): descr = op.getdescr() assert isinstance(descr, AbstractFailDescr) - #if hasattr(op, 'getfailargs'): - # print 'Failargs: ', op.getfailargs() + if hasattr(op, 'getfailargs'): + print 'Failargs: ', op.getfailargs() self.mc.ensure_can_fit(self.guard_size) - self.mc.ADD_ri(r.pc.value, r.pc.value, self.guard_size, cond=fcond) + self.mc.ADD_ri(r.pc.value, r.pc.value, self.guard_size-PC_OFFSET, cond=fcond) descr._arm_guard_code = self.mc.curraddr() self.mc.PUSH([reg.value for reg in r.caller_resp]) @@ -200,7 +200,6 @@ memaddr = self._gen_path_to_exit_path(op, op.getfailargs(), regalloc) descr._failure_recovery_code = memaddr regalloc.possibly_free_vars_for_op(op) - self.mc.NOP() return c.AL def emit_op_guard_true(self, op, regalloc, fcond): From fijal at codespeak.net Thu Dec 2 18:41:56 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 18:41:56 +0100 (CET) Subject: [pypy-svn] r79759 - pypy/branch/out-of-line-guards/pypy/interpreter Message-ID: <20101202174156.CC6E9282BF1@codespeak.net> Author: fijal Date: Thu Dec 2 18:41:55 2010 New Revision: 79759 Modified: pypy/branch/out-of-line-guards/pypy/interpreter/function.py Log: Those fields are actually immutable Modified: pypy/branch/out-of-line-guards/pypy/interpreter/function.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/function.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/function.py Thu Dec 2 18:41:55 2010 @@ -28,6 +28,8 @@ can_change_code = True + _immutable_fields_ = ['w_func_globals', 'closure'] + def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, forcename=None): self.space = space From fijal at codespeak.net Thu Dec 2 18:43:30 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 18:43:30 +0100 (CET) Subject: [pypy-svn] r79760 - pypy/branch/out-of-line-guards/pypy/interpreter Message-ID: <20101202174330.D737A282B90@codespeak.net> Author: fijal Date: Thu Dec 2 18:43:29 2010 New Revision: 79760 Modified: pypy/branch/out-of-line-guards/pypy/interpreter/typedef.py Log: make typedef a newstyle class Modified: pypy/branch/out-of-line-guards/pypy/interpreter/typedef.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/typedef.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/typedef.py Thu Dec 2 18:43:29 2010 @@ -12,7 +12,7 @@ from pypy.rlib.objectmodel import instantiate, compute_identity_hash, specialize from pypy.rlib.jit import hint -class TypeDef: +class TypeDef(object): def __init__(self, __name, __base=None, **rawdict): "NOT_RPYTHON: initialization-time only" self.name = __name From fijal at codespeak.net Thu Dec 2 20:30:00 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 20:30:00 +0100 (CET) Subject: [pypy-svn] r79761 - pypy/trunk/pypy/objspace/std Message-ID: <20101202193000.9A2F9282B9E@codespeak.net> Author: fijal Date: Thu Dec 2 20:29:57 2010 New Revision: 79761 Modified: pypy/trunk/pypy/objspace/std/mapdict.py Log: Use jit_invariant_field for ModuleCell Modified: pypy/trunk/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/mapdict.py (original) +++ pypy/trunk/pypy/objspace/std/mapdict.py Thu Dec 2 20:29:57 2010 @@ -1,4 +1,4 @@ -from pypy.rlib import jit, objectmodel, debug +from pypy.rlib import jit, objectmodel, debug, dont_look_inside from pypy.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root @@ -632,6 +632,7 @@ def _clear_fields(self): self.w_obj = None + @dont_look_inside def _as_rdict(self): self.initialize_as_rdict() space = self.space @@ -640,7 +641,6 @@ self._clear_fields() return self - def materialize_r_dict(space, obj, w_d): map = obj._get_mapdict_map() assert obj.getdict() is w_d From fijal at codespeak.net Thu Dec 2 20:34:35 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 20:34:35 +0100 (CET) Subject: [pypy-svn] r79762 - pypy/trunk/pypy/objspace/std Message-ID: <20101202193435.933CD282BF4@codespeak.net> Author: fijal Date: Thu Dec 2 20:34:33 2010 New Revision: 79762 Modified: pypy/trunk/pypy/objspace/std/mapdict.py Log: revert. wrong branch wrong checkin Modified: pypy/trunk/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/trunk/pypy/objspace/std/mapdict.py (original) +++ pypy/trunk/pypy/objspace/std/mapdict.py Thu Dec 2 20:34:33 2010 @@ -1,4 +1,4 @@ -from pypy.rlib import jit, objectmodel, debug, dont_look_inside +from pypy.rlib import jit, objectmodel, debug from pypy.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root @@ -632,7 +632,6 @@ def _clear_fields(self): self.w_obj = None - @dont_look_inside def _as_rdict(self): self.initialize_as_rdict() space = self.space @@ -641,6 +640,7 @@ self._clear_fields() return self + def materialize_r_dict(space, obj, w_d): map = obj._get_mapdict_map() assert obj.getdict() is w_d From fijal at codespeak.net Thu Dec 2 20:34:57 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 20:34:57 +0100 (CET) Subject: [pypy-svn] r79763 - pypy/branch/out-of-line-guards/pypy/objspace/std Message-ID: <20101202193457.ACF0C282BF5@codespeak.net> Author: fijal Date: Thu Dec 2 20:34:55 2010 New Revision: 79763 Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/celldict.py Log: jit_invariant_fields on ModuleCell Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/celldict.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/celldict.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/celldict.py Thu Dec 2 20:34:55 2010 @@ -8,6 +8,8 @@ from pypy.rlib import jit class ModuleCell(object): + _jit_invariant_fields_ = ['w_value'] + def __init__(self, w_value=None): self.w_value = w_value @@ -47,9 +49,11 @@ else: self._as_rdict().impl_fallback_setitem(w_key, w_value) + @jit.dont_look_inside def impl_setitem_str(self, name, w_value): self.getcell(name, True).w_value = w_value + @jit.dont_look_inside def impl_delitem(self, w_key): space = self.space w_key_type = space.type(w_key) From fijal at codespeak.net Thu Dec 2 22:17:23 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 22:17:23 +0100 (CET) Subject: [pypy-svn] r79764 - in pypy/branch/out-of-line-guards/pypy/jit/metainterp: optimizeopt test Message-ID: <20101202211723.DBAD0282B90@codespeak.net> Author: fijal Date: Thu Dec 2 22:17:20 2010 New Revision: 79764 Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/optimizer.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizeopt.py Log: A lot of fighting and a simple optimization (test was written, just not implemented) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/optimizer.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/optimizer.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/optimizer.py Thu Dec 2 22:17:20 2010 @@ -199,6 +199,8 @@ self.interned_refs = self.cpu.ts.new_ref_dict() self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd) self.bool_boxes = {} + self.int_to_bool_nullity = {} # a mapping from bool boxes to + # respective ints, to know their value self.loop_invariant_results = {} self.pure_operations = args_dict() self.producer = {} Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py Thu Dec 2 22:17:20 2010 @@ -196,9 +196,26 @@ self.optimize_guard(op, constbox, emit_operation) def optimize_GUARD_TRUE(self, op): + value = self.getvalue(op.getarg(0)) + v = self.optimizer.int_to_bool_nullity.get(value) + if v: + v, int_is_true = v + if int_is_true: + if not v.is_nonnull(): + v.make_nonnull(len(self.optimizer.newoperations) - 1) + else: + v.make_constant(ConstInt(0)) self.optimize_guard(op, CONST_1) def optimize_GUARD_FALSE(self, op): + value = self.getvalue(op.getarg(0)) + v = self.optimizer.int_to_bool_nullity.get(value) + if v: + v, int_is_true = v + if int_is_true: + v.make_constant(ConstInt(0)) + elif not v.is_nonnull(): + v.make_nonnull(len(self.optimizer.newoperations) - 1) self.optimize_guard(op, CONST_0) def optimize_GUARD_CLASS(self, op): @@ -270,13 +287,18 @@ self.emit_operation(op) def optimize_INT_IS_TRUE(self, op): - if self.getvalue(op.getarg(0)) in self.optimizer.bool_boxes: - self.make_equal_to(op.result, self.getvalue(op.getarg(0))) + v = self.getvalue(op.getarg(0)) + if v in self.optimizer.bool_boxes: + self.make_equal_to(op.result, v) return self._optimize_nullness(op, op.getarg(0), True) + self.optimizer.int_to_bool_nullity[self.getvalue(op.result)] = (v, True) def optimize_INT_IS_ZERO(self, op): + v = self.getvalue(op.getarg(0)) self._optimize_nullness(op, op.getarg(0), False) + self.optimizer.int_to_bool_nullity[self.getvalue(op.result)] = (v, + False) def _optimize_oois_ooisnot(self, op, expect_isnot): value0 = self.getvalue(op.getarg(0)) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizeopt.py Thu Dec 2 22:17:20 2010 @@ -537,7 +537,6 @@ self.optimize_loop(ops, 'Not', expected) def test_int_is_true_is_zero(self): - py.test.skip("XXX implement me") ops = """ [i0] i1 = int_is_true(i0) @@ -554,6 +553,23 @@ """ self.optimize_loop(ops, 'Not', expected) + def test_int_is_true_is_zero2(self): + ops = """ + [i0] + i2 = int_is_zero(i0) + guard_false(i2) [] + i1 = int_is_true(i0) + guard_true(i1) [] + jump(i0) + """ + expected = """ + [i0] + i1 = int_is_zero(i0) + guard_false(i1) [] + jump(i0) + """ + self.optimize_loop(ops, 'Not', expected) + def test_ooisnull_oononnull_2(self): ops = """ [p0] @@ -4543,7 +4559,6 @@ # not obvious, because of the exception UnicodeDecodeError that # can be raised by ll_str2unicode() - ##class TestOOtype(OptimizeOptTest, OOtypeMixin): ## def test_instanceof(self): From fijal at codespeak.net Thu Dec 2 22:24:14 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 2 Dec 2010 22:24:14 +0100 (CET) Subject: [pypy-svn] r79765 - pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt Message-ID: <20101202212414.9F4F8282B90@codespeak.net> Author: fijal Date: Thu Dec 2 22:24:11 2010 New Revision: 79765 Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py Log: RPython fixes Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py Thu Dec 2 22:24:11 2010 @@ -197,7 +197,7 @@ def optimize_GUARD_TRUE(self, op): value = self.getvalue(op.getarg(0)) - v = self.optimizer.int_to_bool_nullity.get(value) + v = self.optimizer.int_to_bool_nullity.get(value, None) if v: v, int_is_true = v if int_is_true: @@ -209,7 +209,7 @@ def optimize_GUARD_FALSE(self, op): value = self.getvalue(op.getarg(0)) - v = self.optimizer.int_to_bool_nullity.get(value) + v = self.optimizer.int_to_bool_nullity.get(value, None) if v: v, int_is_true = v if int_is_true: From fijal at codespeak.net Fri Dec 3 07:49:23 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 3 Dec 2010 07:49:23 +0100 (CET) Subject: [pypy-svn] r79766 - pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt Message-ID: <20101203064923.6DADD282B9D@codespeak.net> Author: fijal Date: Fri Dec 3 07:49:20 2010 New Revision: 79766 Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py Log: rpythonize Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/rewrite.py Fri Dec 3 07:49:20 2010 @@ -197,9 +197,8 @@ def optimize_GUARD_TRUE(self, op): value = self.getvalue(op.getarg(0)) - v = self.optimizer.int_to_bool_nullity.get(value, None) - if v: - v, int_is_true = v + if value in self.optimizer.int_to_bool_nullity: + v, int_is_true = self.optimizer.int_to_bool_nullity[value] if int_is_true: if not v.is_nonnull(): v.make_nonnull(len(self.optimizer.newoperations) - 1) @@ -209,9 +208,8 @@ def optimize_GUARD_FALSE(self, op): value = self.getvalue(op.getarg(0)) - v = self.optimizer.int_to_bool_nullity.get(value, None) - if v: - v, int_is_true = v + if value in self.optimizer.int_to_bool_nullity: + v, int_is_true = self.optimizer.int_to_bool_nullity[value] if int_is_true: v.make_constant(ConstInt(0)) elif not v.is_nonnull(): From antocuni at codespeak.net Fri Dec 3 10:12:46 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Dec 2010 10:12:46 +0100 (CET) Subject: [pypy-svn] r79767 - pypy/branch/jitypes2/lib_pypy/_ctypes Message-ID: <20101203091246.9BC98282B9D@codespeak.net> Author: antocuni Date: Fri Dec 3 10:12:43 2010 New Revision: 79767 Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Log: add support for converting arguments. The first test in test_functions (test_wchar_parm) passes Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Fri Dec 3 10:12:43 2010 @@ -142,7 +142,7 @@ def __call__(self, *args): if self.callable is not None: - assert False, 'TODO' + assert False, 'TODO1' try: res = self.callable(*args) except: @@ -156,7 +156,7 @@ argtypes = self._argtypes_ if self._com_index: - assert False, 'TODO' + assert False, 'TODO2' from ctypes import cast, c_void_p, POINTER thisarg = cast(args[0], POINTER(POINTER(c_void_p))).contents argtypes = [c_void_p] + list(argtypes) @@ -166,29 +166,51 @@ thisarg = None if argtypes is None: - assert False, 'TODO' + assert False, 'TODO3' argtypes = self._guess_argtypes(args) # XXX - #assert False, 'TODO' - #argtypes, argsandobjs = self._wrap_args(argtypes, args) + argtypes, argsandobjs = self._wrap_args(argtypes, args) + newargs = self._unwrap_args(argtypes, argsandobjs) + restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype, thisarg) - #return funcptr(*args) - return funcptr(args[0]) + return funcptr(*newargs) #resbuffer = funcptr(*[arg._buffer for _, arg in argsandobjs]) #return self._build_result(restype, resbuffer, argtypes, argsandobjs) + # XXX: maybe move this to _ffi + from _ffi import types + _typemap = { + 'c' : types.uchar, + 'b' : types.schar, + 'B' : types.uchar, + 'h' : types.sshort, + 'u' : types.ushort, # XXXXXXXX, use cast_type_to_ffitype(lltype.UniChar) + 'H' : types.ushort, + 'i' : types.sint, + 'I' : types.uint, + 'l' : types.slong, + 'L' : types.ulong, + #'q' : cast_type_to_ffitype(rffi.LONGLONG), + #'Q' : cast_type_to_ffitype(rffi.ULONGLONG), + 'f' : types.float, + 'd' : types.double, + 's' : types.pointer, + 'P' : types.pointer, + 'z' : types.pointer, + 'O' : types.pointer, + 'Z' : types.pointer, + } + del types + def _shape_to_ffi_type(self, shape): - from _ffi import types - if shape == 'l': - return types.slong - elif shape == 'd': - return types.double - else: + try: + return self._typemap[shape] + except KeyError: print 'unknown shape %s' % shape - assert False, 'TODO' + assert False, 'TODO5' def _getfuncptr(self, argtypes, restype, thisarg=None): if self._ptr is not None and argtypes is self._argtypes_: @@ -322,6 +344,17 @@ argtypes = list(argtypes) + extra_types return argtypes, wrapped_args + def _unwrap_args(self, argtypes, argsandobjs): + assert len(argtypes) == len(argsandobjs) + newargs = [] + for argtype, (_, arg) in zip(argtypes, argsandobjs): + value = arg.value + if isinstance(value, basestring) and len(value) == 1: + # XXX: who should do this conversion? Maybe _ffi? + value = ord(value) + newargs.append(value) + return newargs + def _build_result(self, restype, resbuffer, argtypes, argsandobjs): """Build the function result: If there is no OUT parameter, return the actual function result From antocuni at codespeak.net Fri Dec 3 10:14:50 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Dec 2010 10:14:50 +0100 (CET) Subject: [pypy-svn] r79768 - pypy/branch/jitypes2/lib_pypy/_ctypes Message-ID: <20101203091450.52045282B9E@codespeak.net> Author: antocuni Date: Fri Dec 3 10:14:48 2010 New Revision: 79768 Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Log: kill this hack. The goal now is to have something that works, we will make it jit-friendly later Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Fri Dec 3 10:14:48 2010 @@ -240,8 +240,6 @@ ffi_argtypes = [self._shape_to_ffi_type(shape) for shape in argshapes] ffi_restype = self._shape_to_ffi_type(resshape) self._ptr = cdll.getfunc(self.name, ffi_argtypes, ffi_restype) - if len(ffi_argtypes) == 1: - self.__class__ = _FuncPtr_1_arg # black magic return self._ptr except AttributeError: if self._flags_ & _rawffi.FUNCFLAG_CDECL: @@ -429,26 +427,3 @@ self._ptr.free() self._ptr = None self._needs_free = False - - -# hack hack - -#import ctypes -class _FuncPtr_1_arg(CFuncPtr): - - def _rollback(self, *args): - self.__class__ = CFuncPtr - return self(*args) - - def __call__(self, arg0): - if self.callable is not None or self._com_index: - return self._rollback(arg0) - - #argtypes, argsandobjs = self._wrap_args(argtypes, args) - - argtypes = self._argtypes_ - restype = self._restype_ - thisarg = None - funcptr = self._getfuncptr(argtypes, restype, thisarg) - return funcptr(arg0) - From david at codespeak.net Fri Dec 3 10:51:56 2010 From: david at codespeak.net (david at codespeak.net) Date: Fri, 3 Dec 2010 10:51:56 +0100 (CET) Subject: [pypy-svn] r79769 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101203095156.195C5282B9E@codespeak.net> Author: david Date: Fri Dec 3 10:51:54 2010 New Revision: 79769 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: Fixes and improvementes for int_sub and int_neg Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Fri Dec 3 10:51:54 2010 @@ -57,34 +57,39 @@ return fcond def emit_op_int_sub(self, op, regalloc, fcond): - # assuming only one argument is constant + #XXX check if neg values are supported for imm values a0 = op.getarg(0) a1 = op.getarg(1) - imm_a0 = isinstance(a0, ConstInt) and (a0.getint() <= 0xFF or -1 * a0.getint() <= 0xFF) - imm_a1 = isinstance(a1, ConstInt) and (a1.getint() <= 0xFF or -1 * a1.getint() <= 0xFF) - l0 = regalloc.make_sure_var_in_reg(a0, imm_fine=imm_a0) - l1 = regalloc.make_sure_var_in_reg(a1, [a0], imm_fine=imm_a1) - res = regalloc.force_allocate_reg(op.result, [a0, a1]) - if imm_a0: + boxes = [a0, a1] + imm_a0 = self._check_imm_arg(a0) + imm_a1 = self._check_imm_arg(a1) + if not imm_a0 and imm_a1: + l0, box = self._ensure_value_is_boxed(a0, regalloc) + l1 = regalloc.make_sure_var_in_reg(a1, [a0]) + boxes.append(box) + elif imm_a0 and not imm_a1: + l0 = regalloc.make_sure_var_in_reg(a0) + l1, box = self._ensure_value_is_boxed(a1, regalloc) + boxes.append(box) + else: + l0, box = self._ensure_value_is_boxed(a0, regalloc) + boxes.append(box) + l1, box = self._ensure_value_is_boxed(a1, regalloc) + boxes.append(box) + res = regalloc.force_allocate_reg(op.result, boxes) + if l0.is_imm(): value = l0.getint() - if value < 0: - # XXX needs a test - self.mc.ADD_ri(res.value, l1.value, -1 * value, s=1) - self.mc.MVN_rr(res.value, l1.value, s=1) - else: - # reverse substract ftw - self.mc.RSB_ri(res.value, l1.value, value, s=1) - elif imm_a1: + assert value >= 0 + # reverse substract ftw + self.mc.RSB_ri(res.value, l1.value, value, s=1) + elif l1.is_imm(): value = l1.getint() - if value < 0: - self.mc.ADD_ri(res.value, l0.value, -1 * value, s=1) - else: - self.mc.SUB_ri(res.value, l0.value, value, s=1) + assert value >= 0 + self.mc.SUB_ri(res.value, l0.value, value, s=1) else: self.mc.SUB_rr(res.value, l0.value, l1.value, s=1) - regalloc.possibly_free_var(a0) - regalloc.possibly_free_var(a1) + regalloc.possibly_free_vars(boxes) regalloc.possibly_free_var(op.result) return fcond @@ -171,9 +176,9 @@ arg = op.getarg(0) resbox = op.result l0 = regalloc.make_sure_var_in_reg(arg) - l1 = regalloc.make_sure_var_in_reg(ConstInt(-1), [arg], imm_fine=False) + self.mc.MVN_ri(r.ip.value, imm=~-1) resloc = regalloc.force_allocate_reg(resbox, [arg]) - self.mc.MUL(resloc.value, l0.value, l1.value) + self.mc.MUL(resloc.value, l0.value, r.ip.value) regalloc.possibly_free_vars([arg, resbox]) return fcond From antocuni at codespeak.net Fri Dec 3 11:04:39 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Dec 2010 11:04:39 +0100 (CET) Subject: [pypy-svn] r79770 - in pypy/branch/jitypes2: . ctypes_configure lib-python lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/distutils/tests lib-python/modified-2.5.2/test lib_pypy lib_pypy/_ctypes lib_pypy/pypy_test pypy pypy/annotation pypy/annotation/test pypy/config pypy/config/test pypy/doc pypy/doc/config pypy/doc/statistic pypy/interpreter pypy/interpreter/test pypy/jit pypy/jit/backend pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/optimizeopt pypy/jit/metainterp/test pypy/jit/tl pypy/jit/tl/spli pypy/jit/tool pypy/jit/tool/test pypy/module/__pypy__ pypy/module/__pypy__/test pypy/module/_minimal_curses pypy/module/_pickle_support pypy/module/array/benchmark pypy/module/array/test pypy/module/binascii pypy/module/binascii/test pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/src pypy/module/cpyext/test pypy/module/fcntl/test pypy/module/imp pypy/module/itertools pypy/module/posix pypy/module/posix/test pypy/module/pyexpat pypy/module/pyexpat/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/sys pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/rlib pypy/rlib/rsre pypy/rlib/rsre/test pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/memory/gc pypy/rpython/memory/gc/test pypy/rpython/memory/gctransform pypy/rpython/module pypy/rpython/ootypesystem pypy/rpython/test pypy/rpython/tool pypy/tool pypy/translator pypy/translator/c pypy/translator/c/gcc pypy/translator/c/src pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/platform pypy/translator/tool site-packages Message-ID: <20101203100439.8C022282BD4@codespeak.net> Author: antocuni Date: Fri Dec 3 11:04:29 2010 New Revision: 79770 Added: pypy/branch/jitypes2/lib-python/modified-2.5.2/test/seq_tests.py - copied unchanged from r79768, pypy/trunk/lib-python/modified-2.5.2/test/seq_tests.py pypy/branch/jitypes2/lib_pypy/pypy_test/hack___pypy__.py - copied unchanged from r79768, pypy/trunk/lib_pypy/pypy_test/hack___pypy__.py pypy/branch/jitypes2/pypy/config/support.py - copied unchanged from r79768, pypy/trunk/pypy/config/support.py pypy/branch/jitypes2/pypy/config/test/test_support.py - copied unchanged from r79768, pypy/trunk/pypy/config/test/test_support.py pypy/branch/jitypes2/pypy/doc/config/objspace.usemodules.binascii.txt - copied unchanged from r79768, pypy/trunk/pypy/doc/config/objspace.usemodules.binascii.txt pypy/branch/jitypes2/pypy/jit/backend/llsupport/asmmemmgr.py - copied unchanged from r79768, pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py pypy/branch/jitypes2/pypy/jit/backend/llsupport/test/test_asmmemmgr.py - copied unchanged from r79768, pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py pypy/branch/jitypes2/pypy/jit/backend/x86/codebuf.py - copied unchanged from r79768, pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/branch/jitypes2/pypy/jit/metainterp/memmgr.py - copied unchanged from r79768, pypy/trunk/pypy/jit/metainterp/memmgr.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_memmgr.py - copied unchanged from r79768, pypy/trunk/pypy/jit/metainterp/test/test_memmgr.py pypy/branch/jitypes2/pypy/jit/tool/cpython.vmrss - copied unchanged from r79768, pypy/trunk/pypy/jit/tool/cpython.vmrss pypy/branch/jitypes2/pypy/jit/tool/test/test_log2gnumeric.py - copied unchanged from r79768, pypy/trunk/pypy/jit/tool/test/test_log2gnumeric.py pypy/branch/jitypes2/pypy/module/__pypy__/interp_debug.py - copied unchanged from r79768, pypy/trunk/pypy/module/__pypy__/interp_debug.py pypy/branch/jitypes2/pypy/module/__pypy__/test/test_debug.py - copied unchanged from r79768, pypy/trunk/pypy/module/__pypy__/test/test_debug.py pypy/branch/jitypes2/pypy/module/binascii/ (props changed) - copied from r79768, pypy/trunk/pypy/module/binascii/ pypy/branch/jitypes2/pypy/module/cpyext/include/structseq.h - copied unchanged from r79768, pypy/trunk/pypy/module/cpyext/include/structseq.h pypy/branch/jitypes2/pypy/module/cpyext/src/structseq.c - copied unchanged from r79768, pypy/trunk/pypy/module/cpyext/src/structseq.c pypy/branch/jitypes2/pypy/module/cpyext/test/test_structseq.py - copied unchanged from r79768, pypy/trunk/pypy/module/cpyext/test/test_structseq.py pypy/branch/jitypes2/pypy/rpython/memory/gc/env.py - copied unchanged from r79768, pypy/trunk/pypy/rpython/memory/gc/env.py pypy/branch/jitypes2/pypy/rpython/memory/gc/test/test_env.py - copied unchanged from r79768, pypy/trunk/pypy/rpython/memory/gc/test/test_env.py pypy/branch/jitypes2/pypy/rpython/memory/gc/test/test_inspector.py - copied unchanged from r79768, pypy/trunk/pypy/rpython/memory/gc/test/test_inspector.py pypy/branch/jitypes2/pypy/tool/debug_print.py - copied unchanged from r79768, pypy/trunk/pypy/tool/debug_print.py pypy/branch/jitypes2/pypy/translator/platform/freebsd.py - copied unchanged from r79768, pypy/trunk/pypy/translator/platform/freebsd.py Removed: pypy/branch/jitypes2/pypy/doc/config/translation.jit_debug.txt pypy/branch/jitypes2/pypy/translator/platform/freebsd7.py Modified: pypy/branch/jitypes2/ (props changed) pypy/branch/jitypes2/ctypes_configure/configure.py pypy/branch/jitypes2/lib-python/conftest.py pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/msvccompiler.py pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py pypy/branch/jitypes2/lib_pypy/_ctypes/array.py pypy/branch/jitypes2/lib_pypy/_ctypes/basics.py pypy/branch/jitypes2/lib_pypy/_ctypes/builtin.py pypy/branch/jitypes2/lib_pypy/_ctypes/function.py pypy/branch/jitypes2/lib_pypy/_ctypes/pointer.py pypy/branch/jitypes2/lib_pypy/_ctypes/primitive.py pypy/branch/jitypes2/lib_pypy/_ctypes/structure.py pypy/branch/jitypes2/lib_pypy/_hashlib.py pypy/branch/jitypes2/lib_pypy/_locale.py pypy/branch/jitypes2/lib_pypy/_marshal.py pypy/branch/jitypes2/lib_pypy/_minimal_curses.py pypy/branch/jitypes2/lib_pypy/_pypy_interact.py pypy/branch/jitypes2/lib_pypy/binascii.py pypy/branch/jitypes2/lib_pypy/cPickle.py pypy/branch/jitypes2/lib_pypy/cmath.py pypy/branch/jitypes2/lib_pypy/ctypes_support.py pypy/branch/jitypes2/lib_pypy/grp.py pypy/branch/jitypes2/lib_pypy/hashlib.py pypy/branch/jitypes2/lib_pypy/itertools.py pypy/branch/jitypes2/lib_pypy/msvcrt.py pypy/branch/jitypes2/lib_pypy/pwd.py pypy/branch/jitypes2/lib_pypy/pyexpat.py pypy/branch/jitypes2/lib_pypy/pypy_test/test_hashlib.py pypy/branch/jitypes2/lib_pypy/pypy_test/test_structseq.py pypy/branch/jitypes2/lib_pypy/readline.py pypy/branch/jitypes2/lib_pypy/resource.py pypy/branch/jitypes2/lib_pypy/syslog.py pypy/branch/jitypes2/pypy/ (props changed) pypy/branch/jitypes2/pypy/annotation/annrpython.py pypy/branch/jitypes2/pypy/annotation/bookkeeper.py pypy/branch/jitypes2/pypy/annotation/description.py pypy/branch/jitypes2/pypy/annotation/listdef.py pypy/branch/jitypes2/pypy/annotation/model.py pypy/branch/jitypes2/pypy/annotation/specialize.py pypy/branch/jitypes2/pypy/annotation/test/test_annrpython.py pypy/branch/jitypes2/pypy/annotation/unaryop.py pypy/branch/jitypes2/pypy/config/pypyoption.py pypy/branch/jitypes2/pypy/config/translationoption.py pypy/branch/jitypes2/pypy/conftest.py pypy/branch/jitypes2/pypy/doc/cpython_differences.txt pypy/branch/jitypes2/pypy/doc/faq.txt pypy/branch/jitypes2/pypy/doc/index.txt pypy/branch/jitypes2/pypy/doc/release-1.4.0.txt pypy/branch/jitypes2/pypy/doc/sprint-reports.txt pypy/branch/jitypes2/pypy/doc/statistic/release_dates.dat pypy/branch/jitypes2/pypy/doc/statistic/sprint_dates.dat pypy/branch/jitypes2/pypy/interpreter/baseobjspace.py pypy/branch/jitypes2/pypy/interpreter/function.py pypy/branch/jitypes2/pypy/interpreter/gateway.py pypy/branch/jitypes2/pypy/interpreter/generator.py pypy/branch/jitypes2/pypy/interpreter/pycode.py pypy/branch/jitypes2/pypy/interpreter/test/test_gateway.py pypy/branch/jitypes2/pypy/jit/backend/detect_cpu.py pypy/branch/jitypes2/pypy/jit/backend/llgraph/llimpl.py pypy/branch/jitypes2/pypy/jit/backend/llgraph/runner.py pypy/branch/jitypes2/pypy/jit/backend/llsupport/gc.py pypy/branch/jitypes2/pypy/jit/backend/llsupport/llmodel.py pypy/branch/jitypes2/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/jitypes2/pypy/jit/backend/model.py pypy/branch/jitypes2/pypy/jit/backend/test/runner_test.py pypy/branch/jitypes2/pypy/jit/backend/test/test_random.py pypy/branch/jitypes2/pypy/jit/backend/x86/assembler.py pypy/branch/jitypes2/pypy/jit/backend/x86/regalloc.py pypy/branch/jitypes2/pypy/jit/backend/x86/regloc.py pypy/branch/jitypes2/pypy/jit/backend/x86/runner.py pypy/branch/jitypes2/pypy/jit/backend/x86/rx86.py pypy/branch/jitypes2/pypy/jit/backend/x86/support.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_runner.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_zmath.py pypy/branch/jitypes2/pypy/jit/backend/x86/valgrind.py pypy/branch/jitypes2/pypy/jit/codewriter/call.py pypy/branch/jitypes2/pypy/jit/codewriter/jtransform.py pypy/branch/jitypes2/pypy/jit/codewriter/support.py pypy/branch/jitypes2/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/jitypes2/pypy/jit/codewriter/test/test_list.py pypy/branch/jitypes2/pypy/jit/codewriter/test/test_void_list.py pypy/branch/jitypes2/pypy/jit/conftest.py pypy/branch/jitypes2/pypy/jit/metainterp/compile.py pypy/branch/jitypes2/pypy/jit/metainterp/history.py pypy/branch/jitypes2/pypy/jit/metainterp/jitprof.py pypy/branch/jitypes2/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/branch/jitypes2/pypy/jit/metainterp/pyjitpl.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_basic.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_compile.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_list.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_logger.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_recursive.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/jitypes2/pypy/jit/metainterp/warmspot.py pypy/branch/jitypes2/pypy/jit/metainterp/warmstate.py pypy/branch/jitypes2/pypy/jit/tl/spli/interpreter.py pypy/branch/jitypes2/pypy/jit/tl/tl.py pypy/branch/jitypes2/pypy/jit/tool/jitoutput.py pypy/branch/jitypes2/pypy/jit/tool/log-template.gnumeric pypy/branch/jitypes2/pypy/jit/tool/log2gnumeric.py pypy/branch/jitypes2/pypy/jit/tool/test/test_jitoutput.py pypy/branch/jitypes2/pypy/module/__pypy__/__init__.py pypy/branch/jitypes2/pypy/module/__pypy__/interp_magic.py pypy/branch/jitypes2/pypy/module/__pypy__/test/test_special.py pypy/branch/jitypes2/pypy/module/_minimal_curses/__init__.py pypy/branch/jitypes2/pypy/module/_pickle_support/maker.py pypy/branch/jitypes2/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/jitypes2/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/jitypes2/pypy/module/binascii/test/ (props changed) pypy/branch/jitypes2/pypy/module/cpyext/api.py pypy/branch/jitypes2/pypy/module/cpyext/cdatetime.py pypy/branch/jitypes2/pypy/module/cpyext/include/tupleobject.h pypy/branch/jitypes2/pypy/module/cpyext/sequence.py pypy/branch/jitypes2/pypy/module/cpyext/slotdefs.py pypy/branch/jitypes2/pypy/module/cpyext/state.py pypy/branch/jitypes2/pypy/module/cpyext/stubs.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_arraymodule.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_cpyext.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_datetime.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_sequence.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_tupleobject.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_typeobject.py pypy/branch/jitypes2/pypy/module/cpyext/tupleobject.py pypy/branch/jitypes2/pypy/module/cpyext/typeobject.py pypy/branch/jitypes2/pypy/module/fcntl/test/test_fcntl.py pypy/branch/jitypes2/pypy/module/imp/importing.py pypy/branch/jitypes2/pypy/module/itertools/interp_itertools.py pypy/branch/jitypes2/pypy/module/posix/__init__.py pypy/branch/jitypes2/pypy/module/posix/interp_posix.py pypy/branch/jitypes2/pypy/module/posix/test/test_posix2.py pypy/branch/jitypes2/pypy/module/pyexpat/interp_pyexpat.py pypy/branch/jitypes2/pypy/module/pyexpat/test/test_parser.py pypy/branch/jitypes2/pypy/module/pypyjit/policy.py pypy/branch/jitypes2/pypy/module/pypyjit/test/test_policy.py pypy/branch/jitypes2/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/jitypes2/pypy/module/sys/__init__.py pypy/branch/jitypes2/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py pypy/branch/jitypes2/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py pypy/branch/jitypes2/pypy/objspace/std/mapdict.py pypy/branch/jitypes2/pypy/objspace/std/objspace.py pypy/branch/jitypes2/pypy/rlib/debug.py pypy/branch/jitypes2/pypy/rlib/jit.py pypy/branch/jitypes2/pypy/rlib/libffi.py pypy/branch/jitypes2/pypy/rlib/rarithmetic.py pypy/branch/jitypes2/pypy/rlib/rdynload.py pypy/branch/jitypes2/pypy/rlib/rerased.py (contents, props changed) pypy/branch/jitypes2/pypy/rlib/rsre/rsre_core.py pypy/branch/jitypes2/pypy/rlib/rsre/test/test_zjit.py pypy/branch/jitypes2/pypy/rlib/test/test_debug.py pypy/branch/jitypes2/pypy/rlib/test/test_libffi.py pypy/branch/jitypes2/pypy/rlib/test/test_rerased.py (props changed) pypy/branch/jitypes2/pypy/rpython/extfunc.py pypy/branch/jitypes2/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/jitypes2/pypy/rpython/lltypesystem/llmemory.py pypy/branch/jitypes2/pypy/rpython/lltypesystem/rbuiltin.py pypy/branch/jitypes2/pypy/rpython/lltypesystem/rclass.py pypy/branch/jitypes2/pypy/rpython/lltypesystem/rdict.py pypy/branch/jitypes2/pypy/rpython/lltypesystem/rpbc.py pypy/branch/jitypes2/pypy/rpython/memory/gc/base.py pypy/branch/jitypes2/pypy/rpython/memory/gc/generation.py pypy/branch/jitypes2/pypy/rpython/memory/gc/inspector.py pypy/branch/jitypes2/pypy/rpython/memory/gc/markcompact.py pypy/branch/jitypes2/pypy/rpython/memory/gc/minimark.py pypy/branch/jitypes2/pypy/rpython/memory/gc/semispace.py pypy/branch/jitypes2/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/jitypes2/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/jitypes2/pypy/rpython/memory/gctransform/boehm.py pypy/branch/jitypes2/pypy/rpython/memory/gctransform/framework.py pypy/branch/jitypes2/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/jitypes2/pypy/rpython/memory/gctransform/support.py pypy/branch/jitypes2/pypy/rpython/module/ll_os.py pypy/branch/jitypes2/pypy/rpython/module/ll_time.py pypy/branch/jitypes2/pypy/rpython/ootypesystem/rpbc.py pypy/branch/jitypes2/pypy/rpython/rlist.py pypy/branch/jitypes2/pypy/rpython/rpbc.py pypy/branch/jitypes2/pypy/rpython/test/test_rclass.py pypy/branch/jitypes2/pypy/rpython/test/test_rint.py pypy/branch/jitypes2/pypy/rpython/test/test_rlist.py pypy/branch/jitypes2/pypy/rpython/tool/rffi_platform.py pypy/branch/jitypes2/pypy/tool/ansi_print.py pypy/branch/jitypes2/pypy/tool/logparser.py pypy/branch/jitypes2/pypy/translator/c/funcgen.py pypy/branch/jitypes2/pypy/translator/c/gcc/trackgcroot.py pypy/branch/jitypes2/pypy/translator/c/src/debug_alloc.h pypy/branch/jitypes2/pypy/translator/c/src/debug_print.h pypy/branch/jitypes2/pypy/translator/c/src/mem.h pypy/branch/jitypes2/pypy/translator/c/test/test_newgc.py pypy/branch/jitypes2/pypy/translator/c/test/test_standalone.py pypy/branch/jitypes2/pypy/translator/driver.py pypy/branch/jitypes2/pypy/translator/goal/app_main.py pypy/branch/jitypes2/pypy/translator/goal/test2/test_app_main.py pypy/branch/jitypes2/pypy/translator/platform/__init__.py pypy/branch/jitypes2/pypy/translator/platform/darwin.py pypy/branch/jitypes2/pypy/translator/tool/cbuild.py pypy/branch/jitypes2/pypy/translator/tool/reftracker.py pypy/branch/jitypes2/site-packages/ (props changed) Log: merge from trunk: svn merge http://codespeak.net/svn/pypy/trunk -r79389:HEAD in particular, this merges the _ctypes changes that amaury did on the fast-forward branch and then merged on trunk Modified: pypy/branch/jitypes2/ctypes_configure/configure.py ============================================================================== --- pypy/branch/jitypes2/ctypes_configure/configure.py (original) +++ pypy/branch/jitypes2/ctypes_configure/configure.py Fri Dec 3 11:04:29 2010 @@ -559,6 +559,7 @@ C_HEADER = """ #include #include /* for offsetof() */ +#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); Modified: pypy/branch/jitypes2/lib-python/conftest.py ============================================================================== --- pypy/branch/jitypes2/lib-python/conftest.py (original) +++ pypy/branch/jitypes2/lib-python/conftest.py Fri Dec 3 11:04:29 2010 @@ -139,7 +139,7 @@ RegrTest('test_augassign.py', core=True), RegrTest('test_base64.py'), RegrTest('test_bastion.py'), - RegrTest('test_binascii.py'), + RegrTest('test_binascii.py', usemodules='binascii'), RegrTest('test_binhex.py'), Modified: pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/msvccompiler.py Fri Dec 3 11:04:29 2010 @@ -25,17 +25,20 @@ from distutils import log from distutils.util import get_platform -import _winreg - -RegOpenKeyEx = _winreg.OpenKeyEx -RegEnumKey = _winreg.EnumKey -RegEnumValue = _winreg.EnumValue -RegError = _winreg.error - -HKEYS = (_winreg.HKEY_USERS, - _winreg.HKEY_CURRENT_USER, - _winreg.HKEY_LOCAL_MACHINE, - _winreg.HKEY_CLASSES_ROOT) +try: + import _winreg +except ImportError: + pass +else: + RegOpenKeyEx = _winreg.OpenKeyEx + RegEnumKey = _winreg.EnumKey + RegEnumValue = _winreg.EnumValue + RegError = _winreg.error + + HKEYS = (_winreg.HKEY_USERS, + _winreg.HKEY_CURRENT_USER, + _winreg.HKEY_LOCAL_MACHINE, + _winreg.HKEY_CLASSES_ROOT) VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" VSEXPRESS_BASE = r"Software\Microsoft\VCExpress\%0.1f" Modified: pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py ============================================================================== --- pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py (original) +++ pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py Fri Dec 3 11:04:29 2010 @@ -11,11 +11,15 @@ else: self.fail("could not find a suitable manifest") +class MsvcCompilerSimplerTestCase(unittest.TestCase): + def test_import_module(self): + from distutils.msvccompiler import MSVCCompiler + def test_suite(): if sys.platform == 'win32': return unittest.makeSuite(MsvcCompilerTestCase) else: - return unittest.TestSuite([]) + return unittest.makeSuite(MsvcCompilerSimplerTestCase) if __name__ == "__main__": unittest.main(defaultTest="test_suite") Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/array.py Fri Dec 3 11:04:29 2010 @@ -99,6 +99,9 @@ if len(value) > self._length_: raise ValueError("Invalid length") value = self(*value) + elif not isinstance(value, self): + raise TypeError("expected string or Unicode object, %s found" + % (value.__class__.__name__,)) else: if isinstance(value, tuple): if len(value) > self._length_: @@ -107,22 +110,43 @@ return _CDataMeta.from_param(self, value) def array_get_slice_params(self, index): - if index.step is not None: - raise TypeError("3 arg slices not supported (for no reason)") - start = index.start or 0 - stop = index.stop or self._length_ - return start, stop + if hasattr(self, '_length_'): + start, stop, step = index.indices(self._length_) + else: + step = index.step + if step is None: + step = 1 + start = index.start + stop = index.stop + if start is None: + if step > 0: + start = 0 + else: + raise ValueError("slice start is required for step < 0") + if stop is None: + raise ValueError("slice stop is required") + + return start, stop, step def array_slice_setitem(self, index, value): - start, stop = self._get_slice_params(index) - if stop - start != len(value): + start, stop, step = self._get_slice_params(index) + + if ((step < 0 and stop >= start) or + (step > 0 and start >= stop)): + slicelength = 0 + elif step < 0: + slicelength = (stop - start + 1) / step + 1 + else: + slicelength = (stop - start - 1) / step + 1; + + if slicelength != len(value): raise ValueError("Can only assign slices of the same length") - for i in range(start, stop): - self[i] = value[i - start] + for i, j in enumerate(range(start, stop, step)): + self[j] = value[i] def array_slice_getitem(self, index): - start, stop = self._get_slice_params(index) - l = [self[i] for i in range(start, stop)] + start, stop, step = self._get_slice_params(index) + l = [self[i] for i in range(start, stop, step)] letter = getattr(self._type_, '_type_', None) if letter == 'c': return "".join(l) @@ -135,7 +159,8 @@ _ffiargshape = 'P' def __init__(self, *args): - self._buffer = self._ffiarray(self._length_, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(self._length_, autofree=True) for i, arg in enumerate(args): self[i] = arg @@ -162,9 +187,10 @@ self._slice_setitem(index, value) return index = self._fix_index(index) - if ensure_objects(value) is not None: - store_reference(self, index, value._objects) - arg = self._type_._CData_value(value) + cobj = self._type_.from_param(value) + if ensure_objects(cobj) is not None: + store_reference(self, index, cobj._objects) + arg = cobj._get_buffer_value() if self._type_._fficompositesize is None: self._buffer[index] = arg # something more sophisticated, cannot set field directly @@ -183,7 +209,7 @@ return self._length_ def _get_buffer_for_param(self): - return CArgObject(self._buffer.byptr()) + return CArgObject(self, self._buffer.byptr()) def _get_buffer_value(self): return self._buffer.buffer Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/basics.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/basics.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/basics.py Fri Dec 3 11:04:29 2010 @@ -46,21 +46,6 @@ else: return self.from_param(as_parameter) - def _CData_input(self, value): - """Used when data enters into ctypes from user code. 'value' is - some user-specified Python object, which is converted into a _rawffi - array of length 1 containing the same value according to the - type 'self'. - """ - cobj = self.from_param(value) - return cobj, cobj._get_buffer_for_param() - - def _CData_value(self, value): - cobj = self.from_param(value) - # we don't care here if this stuff will live afterwards, as we're - # interested only in value anyway - return cobj._get_buffer_value() - def _CData_output(self, resbuffer, base=None, index=-1): #assert isinstance(resbuffer, _rawffi.ArrayInstance) """Used when data exits ctypes and goes into user code. @@ -92,13 +77,23 @@ """ simple wrapper around buffer, just for the case of freeing it afterwards """ - def __init__(self, buffer): + def __init__(self, obj, buffer): + self._obj = obj self._buffer = buffer def __del__(self): self._buffer.free() self._buffer = None + def __repr__(self): + return '' % (self._obj,) + + def __eq__(self, other): + return self._obj == other + + def __ne__(self, other): + return self._obj != other + class _CData(object): """ The most basic object for all ctypes types """ @@ -128,7 +123,7 @@ return buffer(self._buffer) def _get_b_base(self): - return self._objects + return self._base _b_base_ = property(_get_b_base) _b_needsfree_ = False Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/builtin.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/builtin.py Fri Dec 3 11:04:29 2010 @@ -11,7 +11,8 @@ def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_void_p._CData_value(addr) + cobj = ctypes.c_void_p.from_param(addr) + arg = cobj._get_buffer_value() return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -22,7 +23,8 @@ def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_void_p._CData_value(addr) + cobj = ctypes.c_void_p.from_param(addr) + arg = cobj._get_buffer_value() # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Fri Dec 3 11:04:29 2010 @@ -36,6 +36,7 @@ _argtypes_ = None _restype_ = None + _errcheck_ = None _flags_ = 0 _ffiargshape = 'P' _ffishape = 'P' @@ -54,7 +55,15 @@ return self._argtypes_ def _setargtypes(self, argtypes): self._ptr = None - self._argtypes_ = argtypes + if argtypes is None: + self._argtypes_ = None + else: + for i, argtype in enumerate(argtypes): + if not hasattr(argtype, 'from_param'): + raise TypeError( + "item %d in _argtypes_ has no from_param method" % ( + i + 1,)) + self._argtypes_ = argtypes argtypes = property(_getargtypes, _setargtypes) def _getrestype(self): @@ -73,9 +82,24 @@ del self._restype_ restype = property(_getrestype, _setrestype, _delrestype) + def _geterrcheck(self): + return getattr(self, '_errcheck_', None) + def _seterrcheck(self, errcheck): + if not callable(errcheck): + raise TypeError("The errcheck attribute must be callable") + self._errcheck_ = errcheck + def _delerrcheck(self): + try: + del self._errcheck_ + except AttributeError: + pass + errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck) + def _ffishapes(self, args, restype): argtypes = [arg._ffiargshape for arg in args] if restype is not None: + if not isinstance(restype, _CDataMeta): + raise TypeError("invalid result type for callback function") restype = restype._ffiargshape else: restype = 'O' # void @@ -143,6 +167,7 @@ def __call__(self, *args): if self.callable is not None: assert False, 'TODO1' + args = args[:len(self._argtypes_)] try: res = self.callable(*args) except: @@ -166,19 +191,30 @@ thisarg = None if argtypes is None: - assert False, 'TODO3' - argtypes = self._guess_argtypes(args) - - # XXX - argtypes, argsandobjs = self._wrap_args(argtypes, args) - newargs = self._unwrap_args(argtypes, argsandobjs) + argtypes = [] + args = self._convert_args(argtypes, args) + argtypes = [type(arg) for arg in args] + newargs = self._unwrap_args(argtypes, args) - restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype, thisarg) - return funcptr(*newargs) - #resbuffer = funcptr(*[arg._buffer for _, arg in argsandobjs]) - #return self._build_result(restype, resbuffer, argtypes, argsandobjs) + result = funcptr(*newargs) + ## resbuffer = funcptr(*[arg._get_buffer_for_param()._buffer + ## for arg in args]) + ## result = self._build_result(restype, resbuffer, argtypes, args) + + # The 'errcheck' protocol + if self._errcheck_: + v = self._errcheck_(result, self, args) + # If the errcheck funtion failed, let it throw + # If the errcheck function returned callargs unchanged, + # continue normal processing. + # If the errcheck function returned something else, + # use that as result. + if v is not args: + result = v + + return result # XXX: maybe move this to _ffi from _ffi import types @@ -212,6 +248,7 @@ print 'unknown shape %s' % shape assert False, 'TODO5' + def _getfuncptr(self, argtypes, restype, thisarg=None): if self._ptr is not None and argtypes is self._argtypes_: return self._ptr @@ -258,31 +295,33 @@ raise @staticmethod - def _guess_argtypes(args): + def _conv_param(argtype, arg, index): from ctypes import c_char_p, c_wchar_p, c_void_p, c_int - res = [] - for arg in args: - if hasattr(arg, '_as_parameter_'): - arg = arg._as_parameter_ - if isinstance(arg, str): - res.append(c_char_p) - elif isinstance(arg, unicode): - res.append(c_wchar_p) - elif isinstance(arg, _CData): - res.append(type(arg)) - elif arg is None: - res.append(c_void_p) - #elif arg == 0: - # res.append(c_void_p) - elif isinstance(arg, (int, long)): - res.append(c_int) - else: - raise TypeError("Don't know how to handle %s" % (arg,)) - return res + if argtype is not None: + arg = argtype.from_param(arg) + if hasattr(arg, '_as_parameter_'): + arg = arg._as_parameter_ + + if isinstance(arg, _CData): + # The usual case when argtype is defined + cobj = arg + elif isinstance(arg, str): + cobj = c_char_p(arg) + elif isinstance(arg, unicode): + cobj = c_wchar_p(arg) + elif arg is None: + cobj = c_void_p() + elif isinstance(arg, (int, long)): + cobj = c_int(arg) + else: + raise TypeError("Don't know how to handle %s" % (arg,)) + + return cobj - def _wrap_args(self, argtypes, args): + def _convert_args(self, argtypes, args): wrapped_args = [] consumed = 0 + for i, argtype in enumerate(argtypes): defaultvalue = None if i > 0 and self._paramflags is not None: @@ -309,7 +348,7 @@ val = defaultvalue if val is None: val = 0 - wrapped = argtype._CData_input(val) + wrapped = self._conv_param(argtype, val, consumed) wrapped_args.append(wrapped) continue else: @@ -324,28 +363,27 @@ raise TypeError("Not enough arguments") try: - wrapped = argtype._CData_input(arg) - except (UnicodeError, TypeError), e: + wrapped = self._conv_param(argtype, arg, consumed) + except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) wrapped_args.append(wrapped) consumed += 1 if len(wrapped_args) < len(args): extra = args[len(wrapped_args):] - extra_types = self._guess_argtypes(extra) - for arg, argtype in zip(extra, extra_types): + argtypes = list(argtypes) + for i, arg in enumerate(extra): try: - wrapped = argtype._CData_input(arg) - except (UnicodeError, TypeError), e: + wrapped = self._conv_param(None, arg, i) + except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) wrapped_args.append(wrapped) - argtypes = list(argtypes) + extra_types - return argtypes, wrapped_args + return wrapped_args - def _unwrap_args(self, argtypes, argsandobjs): - assert len(argtypes) == len(argsandobjs) + def _unwrap_args(self, argtypes, args): + assert len(argtypes) == len(args) newargs = [] - for argtype, (_, arg) in zip(argtypes, argsandobjs): + for argtype, arg in zip(argtypes, args): value = arg.value if isinstance(value, basestring) and len(value) == 1: # XXX: who should do this conversion? Maybe _ffi? @@ -364,7 +402,7 @@ if self._com_index: if resbuffer[0] & 0x80000000: raise get_com_error(resbuffer[0], - self._com_iid, argsandobjs[0][0]) + self._com_iid, argsandobjs[0]) else: retval = int(resbuffer[0]) elif restype is not None: @@ -383,8 +421,8 @@ results = [] if self._paramflags: - for argtype, (obj, _), paramflag in zip(argtypes[1:], argsandobjs[1:], - self._paramflags): + for argtype, obj, paramflag in zip(argtypes[1:], argsandobjs[1:], + self._paramflags): if len(paramflag) == 2: idlflag, name = paramflag elif len(paramflag) == 3: Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/pointer.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/pointer.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/pointer.py Fri Dec 3 11:04:29 2010 @@ -1,7 +1,8 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, cdata_from_address -from _ctypes.basics import sizeof, byref, keepalive_key +from _ctypes.basics import keepalive_key, store_reference, ensure_objects +from _ctypes.basics import sizeof, byref from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem @@ -55,7 +56,8 @@ def set_type(self, TP): ffiarray = _rawffi.Array('P') def __init__(self, value=None): - self._buffer = ffiarray(1, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = ffiarray(1, autofree=True) if value is not None: self.contents = value self._ffiarray = ffiarray @@ -99,7 +101,10 @@ return self._type_._CData_output(self._subarray(index), self, index) def __setitem__(self, index, value): - self._subarray(index)[0] = self._type_._CData_value(value) + cobj = self._type_.from_param(value) + if ensure_objects(cobj) is not None: + store_reference(self, index, cobj._objects) + self._subarray(index)[0] = cobj._get_buffer_value() def __nonzero__(self): return self._buffer[0] != 0 @@ -110,29 +115,32 @@ if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): raise TypeError("cast() argument 2 must be a pointer type, not %s" % (tp,)) - if isinstance(obj, Array): - ptr = tp.__new__(tp) - ptr._buffer = tp._ffiarray(1, autofree=True) - ptr._buffer[0] = obj._buffer - return ptr if isinstance(obj, (int, long)): result = tp() result._buffer[0] = obj return result - if obj is None: + elif obj is None: result = tp() return result - if not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): + elif isinstance(obj, Array): + ptr = tp.__new__(tp) + ptr._buffer = tp._ffiarray(1, autofree=True) + ptr._buffer[0] = obj._buffer + result = ptr + elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) - result = tp() + else: + result = tp() + result._buffer[0] = obj._buffer[0] # The casted objects '_objects' member: - # It must certainly contain the source objects one. + # From now on, both objects will use the same dictionary + # It must certainly contain the source objects # It must contain the source object itself. if obj._ensure_objects() is not None: - result._objects = {keepalive_key(0): obj._objects, - keepalive_key(1): obj} + result._objects = obj._objects + if isinstance(obj._objects, dict): + result._objects[id(obj)] = obj - result._buffer[0] = obj._buffer[0] return result Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/primitive.py Fri Dec 3 11:04:29 2010 @@ -132,8 +132,8 @@ ConvMode.errors) #self._objects = value array = _rawffi.Array('c')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -155,8 +155,8 @@ ConvMode.errors) #self._objects = value array = _rawffi.Array('u')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -174,8 +174,8 @@ def _setvalue(self, value): if isinstance(value, str): array = _rawffi.Array('c')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -271,7 +271,9 @@ def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) - return output.value + if self.__bases__[0] is _SimpleCData: + return output.value + return output def _sizeofinstances(self): return _rawffi.sizeof(self._type_) @@ -287,7 +289,8 @@ _type_ = 'i' def __init__(self, value=DEFAULT_VALUE): - self._buffer = self._ffiarray(1, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(1, autofree=True) if value is not DEFAULT_VALUE: self.value = value @@ -312,7 +315,11 @@ return self.value def __repr__(self): - return "%s(%r)" % (type(self).__name__, self.value) + if type(self).__bases__[0] is _SimpleCData: + return "%s(%r)" % (type(self).__name__, self.value) + else: + return "<%s object at 0x%x>" % (type(self).__name__, + id(self)) def __nonzero__(self): return self._buffer[0] not in (0, '\x00') Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/structure.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/structure.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/structure.py Fri Dec 3 11:04:29 2010 @@ -34,9 +34,11 @@ if not isinstance(tp, _CDataMeta): raise TypeError("Expected CData subclass, got %s" % (tp,)) import ctypes - all_fields = _fields_[:] - for cls in inspect.getmro(superclass): - all_fields += getattr(cls, '_fields_', []) + all_fields = [] + for cls in reversed(inspect.getmro(superclass)): + # The first field comes from the most base class + all_fields.extend(getattr(cls, '_fields_', [])) + all_fields.extend(_fields_) names = [name for name, ctype in all_fields] rawfields = [(name, ctype._ffishape) for name, ctype in all_fields] @@ -168,7 +170,7 @@ def __init__(self, *args, **kwds): if len(args) > len(self._names): - raise TypeError("too many arguments") + raise TypeError("too many initializers") for name, arg in zip(self._names, args): if name in kwds: raise TypeError("duplicate value for argument %r" % ( Modified: pypy/branch/jitypes2/lib_pypy/_hashlib.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_hashlib.py (original) +++ pypy/branch/jitypes2/lib_pypy/_hashlib.py Fri Dec 3 11:04:29 2010 @@ -148,21 +148,29 @@ return hash(ctx, name) # shortcut functions +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + at builtinify def openssl_md5(string=''): return new('md5', string) + at builtinify def openssl_sha1(string=''): return new('sha1', string) + at builtinify def openssl_sha224(string=''): return new('sha224', string) + at builtinify def openssl_sha256(string=''): return new('sha256', string) + at builtinify def openssl_sha384(string=''): return new('sha384', string) + at builtinify def openssl_sha512(string=''): return new('sha512', string) - Modified: pypy/branch/jitypes2/lib_pypy/_locale.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_locale.py (original) +++ pypy/branch/jitypes2/lib_pypy/_locale.py Fri Dec 3 11:04:29 2010 @@ -11,6 +11,9 @@ # load the platform-specific cache made by running locale.ctc.py from ctypes_config_cache._locale_cache import * +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + # Ubuntu Gusty i386 structure class lconv(Structure): @@ -158,6 +161,7 @@ ul = ''.join(ul) string.letters = ul + at builtinify def setlocale(category, locale=None): "(integer,string=None) -> string. Activates/queries locale processing." if locale: @@ -182,6 +186,7 @@ groups.append(0) return groups + at builtinify def localeconv(): "() -> dict. Returns numeric and monetary locale-specific parameters." @@ -215,6 +220,7 @@ } return result + at builtinify def strcoll(s1, s2): "string,string -> int. Compares two strings according to the locale." @@ -233,6 +239,7 @@ # Collate the strings. return _wcscoll(s1, s2) + at builtinify def strxfrm(s): "string -> string. Returns a string that behaves for cmp locale-aware." @@ -246,6 +253,7 @@ _strxfrm(buf, s, n2) return buf.value + at builtinify def getdefaultlocale(): # TODO: Port code from CPython for Windows and Mac OS raise NotImplementedError() @@ -267,26 +275,31 @@ raise ValueError("unsupported langinfo constant") if HAS_LIBINTL: + @builtinify def gettext(msg): """gettext(msg) -> string Return translation of msg.""" return _gettext(msg) + @builtinify def dgettext(domain, msg): """dgettext(domain, msg) -> string Return translation of msg in domain.""" return _dgettext(domain, msg) + @builtinify def dcgettext(domain, msg, category): """dcgettext(domain, msg, category) -> string Return translation of msg in domain and category.""" return _dcgettext(domain, msg, category) + @builtinify def textdomain(domain): """textdomain(domain) -> string Set the C library's textdomain to domain, returning the new domain.""" return _textdomain(domain) + @builtinify def bindtextdomain(domain, dir): """bindtextdomain(domain, dir) -> string Bind the C library's domain to dir.""" @@ -297,6 +310,7 @@ return dirname if HAS_BIND_TEXTDOMAIN_CODESET: + @builtinify def bind_textdomain_codeset(domain, codeset): """bind_textdomain_codeset(domain, codeset) -> string Bind the C library's domain to codeset.""" Modified: pypy/branch/jitypes2/lib_pypy/_marshal.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_marshal.py (original) +++ pypy/branch/jitypes2/lib_pypy/_marshal.py Fri Dec 3 11:04:29 2010 @@ -6,6 +6,10 @@ import types from _codecs import utf_8_decode, utf_8_encode +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + TYPE_NULL = '0' TYPE_NONE = 'N' TYPE_FALSE = 'F' @@ -645,15 +649,18 @@ version = 1 + at builtinify def dump(x, f, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format m = _Marshaller(f.write) m.dump(x) + at builtinify def load(f): um = _Unmarshaller(f.read) return um.load() + at builtinify def dumps(x, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format buffer = [] @@ -661,6 +668,7 @@ m.dump(x) return ''.join(buffer) + at builtinify def loads(s): um = _FastUnmarshaller(s) return um.load() Modified: pypy/branch/jitypes2/lib_pypy/_minimal_curses.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_minimal_curses.py (original) +++ pypy/branch/jitypes2/lib_pypy/_minimal_curses.py Fri Dec 3 11:04:29 2010 @@ -35,18 +35,24 @@ # ____________________________________________________________ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + at builtinify def setupterm(termstr, fd): err = ctypes.c_int(0) result = clib.setupterm(termstr, fd, ctypes.byref(err)) if result == ERR: raise error("setupterm() failed (err=%d)" % err.value) + at builtinify def tigetstr(cap): result = clib.tigetstr(cap) if ctypes.cast(result, ctypes.c_void_p).value == ERR: return None return ctypes.cast(result, ctypes.c_char_p).value + at builtinify def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) if result is None: Modified: pypy/branch/jitypes2/lib_pypy/_pypy_interact.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_pypy_interact.py (original) +++ pypy/branch/jitypes2/lib_pypy/_pypy_interact.py Fri Dec 3 11:04:29 2010 @@ -4,6 +4,13 @@ def interactive_console(mainmodule=None): + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, 'ps1'): + sys.ps1 = '>>>> ' + if not hasattr(sys, 'ps2'): + sys.ps2 = '.... ' + # try: from _pypy_irc_topic import some_topic text = "And now for something completely different: ``%s''" % ( @@ -15,6 +22,7 @@ print text except ImportError: pass + # try: from pyrepl.simple_interact import check if not check(): Modified: pypy/branch/jitypes2/lib_pypy/binascii.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/binascii.py (original) +++ pypy/branch/jitypes2/lib_pypy/binascii.py Fri Dec 3 11:04:29 2010 @@ -1,3 +1,9 @@ +"""A pure Python implementation of binascii. + +Rather slow and buggy in corner cases. +PyPy provides an RPython version too. +""" + class Error(Exception): pass @@ -277,7 +283,7 @@ if (c > '~' or c == '=' or (header and c == '_') or - (c == '.' and linelen == 0 and (inp == len(data) or + (c == '.' and linelen == 0 and (inp+1 == len(data) or data[inp+1] == '\n' or data[inp+1] == '\r')) or (not istext and (c == '\r' or c == '\n')) or Modified: pypy/branch/jitypes2/lib_pypy/cPickle.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/cPickle.py (original) +++ pypy/branch/jitypes2/lib_pypy/cPickle.py Fri Dec 3 11:04:29 2010 @@ -5,6 +5,10 @@ from pickle import * from pickle import __doc__, __version__, format_version, compatible_formats +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + BadPickleGet = KeyError UnpickleableError = PicklingError @@ -31,9 +35,11 @@ def getvalue(self): return self.__f and self.__f.getvalue() + at builtinify def dump(obj, file, protocol=None): Pickler(file, protocol).dump(obj) + at builtinify def dumps(obj, protocol=None): file = StringIO() Pickler(file, protocol).dump(obj) Modified: pypy/branch/jitypes2/lib_pypy/cmath.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/cmath.py (original) +++ pypy/branch/jitypes2/lib_pypy/cmath.py Fri Dec 3 11:04:29 2010 @@ -7,7 +7,10 @@ import math from math import e, pi - + +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + # constants _one = complex(1., 0.) @@ -24,6 +27,7 @@ return complex(real, imag) + at builtinify def acos(x): """acos(x) @@ -32,6 +36,7 @@ return -(_prodi(log((x+(_i*sqrt((_one-(x*x)))))))) + at builtinify def acosh(x): """acosh(x) @@ -41,6 +46,7 @@ return z+z + at builtinify def asin(x): """asin(x) @@ -52,6 +58,7 @@ return -(_prodi(log((sqrt_1_minus_x_sq+_prodi(x))))) + at builtinify def asinh(x): """asinh(x) @@ -61,6 +68,7 @@ return z+z + at builtinify def atan(x): """atan(x) @@ -69,6 +77,7 @@ return _halfi*log(((_i+x)/(_i-x))) + at builtinify def atanh(x): """atanh(x) @@ -77,6 +86,7 @@ return _half*log((_one+x)/(_one-x)) + at builtinify def cos(x): """cos(x) @@ -88,6 +98,7 @@ return complex(real, imag) + at builtinify def cosh(x): """cosh(x) @@ -99,6 +110,7 @@ return complex(real, imag) + at builtinify def exp(x): """exp(x) @@ -111,6 +123,7 @@ return complex(real, imag) + at builtinify def log(x, base=None): """log(x) @@ -125,6 +138,7 @@ return complex(real, imag) + at builtinify def log10(x): """log10(x) @@ -137,6 +151,7 @@ return complex(real, imag) + at builtinify def sin(x): """sin(x) @@ -148,6 +163,7 @@ return complex(real, imag) + at builtinify def sinh(x): """sinh(x) @@ -159,6 +175,7 @@ return complex(real, imag) + at builtinify def sqrt(x): """sqrt(x) @@ -184,6 +201,7 @@ _sqrt_half = sqrt(_half) + at builtinify def tan(x): """tan(x) @@ -204,6 +222,7 @@ return complex(real, imag) + at builtinify def tanh(x): """tanh(x) Modified: pypy/branch/jitypes2/lib_pypy/ctypes_support.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/ctypes_support.py (original) +++ pypy/branch/jitypes2/lib_pypy/ctypes_support.py Fri Dec 3 11:04:29 2010 @@ -25,7 +25,7 @@ def _where_is_errno(): return standard_c_lib.__errno_location() -elif sys.platform == 'darwin': +elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() Modified: pypy/branch/jitypes2/lib_pypy/grp.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/grp.py (original) +++ pypy/branch/jitypes2/lib_pypy/grp.py Fri Dec 3 11:04:29 2010 @@ -9,6 +9,10 @@ from ctypes import Structure, c_char_p, c_int, POINTER from ctypes_support import standard_c_lib as libc +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + gid_t = c_int class GroupStruct(Structure): @@ -64,6 +68,7 @@ return Group(res.contents.gr_name, res.contents.gr_passwd, res.contents.gr_gid, mem) + at builtinify def getgrgid(gid): res = libc.getgrgid(gid) if not res: @@ -71,6 +76,7 @@ raise KeyError(gid) return _group_from_gstruct(res) + at builtinify def getgrnam(name): if not isinstance(name, str): raise TypeError("expected string") @@ -79,6 +85,7 @@ raise KeyError(name) return _group_from_gstruct(res) + at builtinify def getgrall(): libc.setgrent() lst = [] Modified: pypy/branch/jitypes2/lib_pypy/hashlib.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/hashlib.py (original) +++ pypy/branch/jitypes2/lib_pypy/hashlib.py Fri Dec 3 11:04:29 2010 @@ -50,53 +50,30 @@ 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' """ +import sys try: import _hashlib except ImportError: _hashlib = None -def __get_builtin_constructor(name): - if name in ('SHA1', 'sha1'): - import sha - return sha.new - elif name in ('MD5', 'md5'): - import md5 - return md5.new - elif name in ('SHA256', 'sha256'): - import _sha256 - return _sha256.sha256 - elif name in ('SHA224', 'sha224'): - import _sha256 - return _sha256.sha224 - elif name in ('SHA512', 'sha512'): - import _sha512 - return _sha512.sha512 - elif name in ('SHA384', 'sha384'): - import _sha512 - return _sha512.sha384 - raise ValueError, "unsupported hash type" - def __hash_new(name, string=''): """new(name, string='') - Return a new hashing object using the named algorithm; optionally initialized with a string. """ try: - if _hashlib: - return _hashlib.new(name, string) - except ValueError: - # If the _hashlib module (OpenSSL) doesn't support the named - # hash, try using our builtin implementations. - # This allows for SHA224/256 and SHA384/512 support even though - # the OpenSSL library prior to 0.9.8 doesn't provide them. - pass - - return __get_builtin_constructor(name)(string) + new = __byname[name] + except KeyError: + raise ValueError("unsupported hash type") + return new(string) new = __hash_new -def _setfuncs(): - # use the wrapper of the C implementation +# ____________________________________________________________ + +__byname = {} +def __use_openssl_funcs(): + # use the wrapper of the C implementation sslprefix = 'openssl_' for opensslfuncname, func in vars(_hashlib).items(): if not opensslfuncname.startswith(sslprefix): @@ -106,23 +83,41 @@ # try them all, some may not work due to the OpenSSL # version not supporting that algorithm. func() - # Use the C function directly (very fast) - globals()[funcname] = func + # Use the C function directly (very fast, but with ctypes overhead) + __byname[funcname] = func except ValueError: - try: - # Use the builtin implementation directly (fast) - globals()[funcname] = __get_builtin_constructor(funcname) - except ValueError: - # this one has no builtin implementation, don't define it - pass + pass + +def __use_builtin_funcs(): + # look up the built-in versions (written in Python or RPython), + # and use the fastest one: + # 1. the one in RPython + # 2. the one from openssl (slower due to ctypes calling overhead) + # 3. the one in pure Python + if 'sha1' not in __byname or 'sha' in sys.builtin_module_names: + import sha + __byname['sha1'] = sha.new + if 'md5' not in __byname or 'md5' in sys.builtin_module_names: + import md5 + __byname['md5'] = md5.new + if 'sha256' not in __byname: + import _sha256 + __byname['sha256'] = _sha256.sha256 + if 'sha224' not in __byname: + import _sha256 + __byname['sha224'] = _sha256.sha224 + if 'sha512' not in __byname: + import _sha512 + __byname['sha512'] = _sha512.sha512 + if 'sha384' not in __byname: + import _sha512 + __byname['sha384'] = _sha512.sha384 + +def __export_funcs(): + for key, value in __byname.items(): + globals()[key] = __byname[key.upper()] = value if _hashlib: - _setfuncs() -else: - # lookup the C function to use directly for the named constructors - md5 = __get_builtin_constructor('md5') - sha1 = __get_builtin_constructor('sha1') - sha224 = __get_builtin_constructor('sha224') - sha256 = __get_builtin_constructor('sha256') - sha384 = __get_builtin_constructor('sha384') - sha512 = __get_builtin_constructor('sha512') + __use_openssl_funcs() +__use_builtin_funcs() +__export_funcs() Modified: pypy/branch/jitypes2/lib_pypy/itertools.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/itertools.py (original) +++ pypy/branch/jitypes2/lib_pypy/itertools.py Fri Dec 3 11:04:29 2010 @@ -27,6 +27,9 @@ 'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap', 'takewhile', 'tee'] +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + class chain(object): """Make an iterator that returns elements from the first iterable @@ -565,7 +568,8 @@ def __iter__(self): return self - + + at builtinify def tee(iterable, n=2): """Return n independent iterators from a single iterable. Note : once tee() has made a split, the original iterable Modified: pypy/branch/jitypes2/lib_pypy/msvcrt.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/msvcrt.py (original) +++ pypy/branch/jitypes2/lib_pypy/msvcrt.py Fri Dec 3 11:04:29 2010 @@ -17,6 +17,10 @@ except AttributeError: # we are not on windows raise ImportError +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int] open_osfhandle.restype = ctypes.c_int @@ -34,6 +38,7 @@ _locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] _locking.restype = ctypes.c_int + at builtinify def locking(fd, mode, nbytes): '''lock or unlock a number of bytes in a file.''' rv = _locking(fd, mode, nbytes) Modified: pypy/branch/jitypes2/lib_pypy/pwd.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/pwd.py (original) +++ pypy/branch/jitypes2/lib_pypy/pwd.py Fri Dec 3 11:04:29 2010 @@ -17,6 +17,10 @@ from ctypes_support import standard_c_lib as libc from ctypes import Structure, POINTER, c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + uid_t = c_int gid_t = c_int @@ -79,10 +83,12 @@ _endpwent.argtypes = None _endpwent.restype = None + at builtinify def mkpwent(pw): pw = pw.contents return struct_passwd(pw) + at builtinify def getpwuid(uid): """ getpwuid(uid) -> (pw_name,pw_passwd,pw_uid, @@ -95,6 +101,7 @@ raise KeyError("getpwuid(): uid not found: %s" % uid) return mkpwent(pw) + at builtinify def getpwnam(name): """ getpwnam(name) -> (pw_name,pw_passwd,pw_uid, @@ -109,9 +116,10 @@ raise KeyError("getpwname(): name not found: %s" % name) return mkpwent(pw) + at builtinify def getpwall(): """ - "getpwall() -> list_of_entries + getpwall() -> list_of_entries Return a list of all available password database entries, in arbitrary order. See pwd.__doc__ for more on password database entries. """ Modified: pypy/branch/jitypes2/lib_pypy/pyexpat.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/pyexpat.py (original) +++ pypy/branch/jitypes2/lib_pypy/pyexpat.py Fri Dec 3 11:04:29 2010 @@ -7,6 +7,9 @@ # load the platform-specific cache made by running pyexpat.ctc.py from ctypes_config_cache._pyexpat_cache import * +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + lib = ctypes.CDLL(ctypes.util.find_library('expat')) @@ -425,9 +428,11 @@ new_parser._set_unknown_encoding_handler() return new_parser + at builtinify def ErrorString(errno): return XML_ErrorString(errno)[:200] + at builtinify def ParserCreate(encoding=None, namespace_separator=None, intern=None): if (not isinstance(encoding, str) and not encoding is None): Modified: pypy/branch/jitypes2/lib_pypy/pypy_test/test_hashlib.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/pypy_test/test_hashlib.py (original) +++ pypy/branch/jitypes2/lib_pypy/pypy_test/test_hashlib.py Fri Dec 3 11:04:29 2010 @@ -2,10 +2,20 @@ from ..ctypes_config_cache import rebuild rebuild.rebuild_one('hashlib.ctc.py') +from . import hack___pypy__ from .. import hashlib, _hashlib def test_unicode(): - assert isinstance(hashlib.new('sha1', u'xxx'), _hashlib.hash) + assert isinstance(hashlib.new('sha256', u'xxx'), _hashlib.hash) + +pure_python_version = { + 'md5': 'md5.new', + 'sha1': 'sha.new', + 'sha224': '_sha256.sha224', + 'sha256': '_sha256.sha256', + 'sha384': '_sha512.sha384', + 'sha512': '_sha512.sha512', + } def test_attributes(): for name, expected_size in {'md5': 16, @@ -30,7 +40,9 @@ assert hexdigest == h.hexdigest() # also test the pure Python implementation - h = hashlib.__get_builtin_constructor(name)('') + modname, constructor = pure_python_version[name].split('.') + builder = getattr(__import__(modname), constructor) + h = builder('') assert h.digest_size == expected_size assert h.digestsize == expected_size # Modified: pypy/branch/jitypes2/lib_pypy/pypy_test/test_structseq.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/pypy_test/test_structseq.py (original) +++ pypy/branch/jitypes2/lib_pypy/pypy_test/test_structseq.py Fri Dec 3 11:04:29 2010 @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from .._structseq import * +from .._structseq import structseqfield, structseqtype class mydata: Modified: pypy/branch/jitypes2/lib_pypy/readline.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/readline.py (original) +++ pypy/branch/jitypes2/lib_pypy/readline.py Fri Dec 3 11:04:29 2010 @@ -6,8 +6,4 @@ are only stubs at the moment. """ -# Note that PyPy contains also a built-in module 'readline' which will hide -# this one if compiled in. However the built-in module is incomplete; -# don't use it. - from pyrepl.readline import * Modified: pypy/branch/jitypes2/lib_pypy/resource.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/resource.py (original) +++ pypy/branch/jitypes2/lib_pypy/resource.py Fri Dec 3 11:04:29 2010 @@ -11,6 +11,10 @@ from errno import EINVAL, EPERM import _structseq +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + class error(Exception): pass @@ -77,6 +81,7 @@ ru_nvcsw = _structseq.structseqfield(14) ru_nivcsw = _structseq.structseqfield(15) + at builtinify def rlimit_check_bounds(rlim_cur, rlim_max): if rlim_cur > rlim_t_max: raise ValueError("%d does not fit into rlim_t" % rlim_cur) @@ -89,6 +94,7 @@ ("rlim_max", rlim_t), ) + at builtinify def getrusage(who): ru = _struct_rusage() ret = _getrusage(who, byref(ru)) @@ -116,6 +122,7 @@ ru.ru_nivcsw, )) + at builtinify def getrlimit(resource): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -127,6 +134,7 @@ raise error(errno) return (rlim.rlim_cur, rlim.rlim_max) + at builtinify def setrlimit(resource, rlim): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -143,6 +151,7 @@ else: raise error(errno) + at builtinify def getpagesize(): pagesize = 0 if _getpagesize: Modified: pypy/branch/jitypes2/lib_pypy/syslog.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/syslog.py (original) +++ pypy/branch/jitypes2/lib_pypy/syslog.py Fri Dec 3 11:04:29 2010 @@ -15,6 +15,10 @@ from ctypes_support import standard_c_lib as libc from ctypes import c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + # Real prototype is: # void syslog(int priority, const char *format, ...); # But we also need format ("%s") and one format argument (message) @@ -34,9 +38,11 @@ _setlogmask.argtypes = (c_int,) _setlogmask.restype = c_int + at builtinify def openlog(ident, option, facility): _openlog(ident, option, facility) + at builtinify def syslog(arg1, arg2=None): if arg2 is not None: priority, message = arg1, arg2 @@ -44,15 +50,19 @@ priority, message = LOG_INFO, arg1 _syslog(priority, "%s", message) + at builtinify def closelog(): _closelog() + at builtinify def setlogmask(mask): return _setlogmask(mask) + at builtinify def LOG_MASK(pri): return (1 << pri) + at builtinify def LOG_UPTO(pri): return (1 << (pri + 1)) - 1 Modified: pypy/branch/jitypes2/pypy/annotation/annrpython.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/annrpython.py (original) +++ pypy/branch/jitypes2/pypy/annotation/annrpython.py Fri Dec 3 11:04:29 2010 @@ -145,7 +145,7 @@ classdef.add_source_for_attribute(attr, classdef.classdesc) self.bookkeeper assert isinstance(s_result, annmodel.SomePBC) - olddesc = s_result.descriptions.iterkeys().next() + olddesc = s_result.any_description() desc = olddesc.bind_self(classdef) args = self.bookkeeper.build_args("simple_call", args_s[:]) desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc], Modified: pypy/branch/jitypes2/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/jitypes2/pypy/annotation/bookkeeper.py Fri Dec 3 11:04:29 2010 @@ -262,7 +262,7 @@ args_s, s_result) def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result): - descs = s_callable.descriptions.keys() + descs = list(s_callable.descriptions) if not descs: return family = descs[0].getcallfamily() @@ -590,7 +590,7 @@ assert s_attr.is_constant() attr = s_attr.const - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue @@ -633,7 +633,7 @@ """Analyse a call to a SomePBC() with the given args (list of annotations). """ - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue first = descs[0] Modified: pypy/branch/jitypes2/pypy/annotation/description.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/description.py (original) +++ pypy/branch/jitypes2/pypy/annotation/description.py Fri Dec 3 11:04:29 2010 @@ -636,6 +636,24 @@ return self return None + def maybe_return_immutable_list(self, attr, s_result): + # hack: 'x.lst' where lst is listed in _immutable_fields_ as 'lst[*]' + # should really return an immutable list as a result. Implemented + # by changing the result's annotation (but not, of course, doing an + # actual copy in the rtyper). Tested in pypy.rpython.test.test_rlist, + # test_immutable_list_out_of_instance. + search = '%s[*]' % (attr,) + cdesc = self + while cdesc is not None: + if '_immutable_fields_' in cdesc.classdict: + if search in cdesc.classdict['_immutable_fields_'].value: + s_result.listdef.never_resize() + s_copy = s_result.listdef.offspring() + s_copy.listdef.mark_as_immutable() + return s_copy + cdesc = cdesc.basedesc + return s_result # common case + def consider_call_site(bookkeeper, family, descs, args, s_result): from pypy.annotation.model import SomeInstance, SomePBC, s_None if len(descs) == 1: @@ -654,7 +672,7 @@ if isinstance(s_init, SomePBC): assert len(s_init.descriptions) == 1, ( "unexpected dynamic __init__?") - initfuncdesc = s_init.descriptions.keys()[0] + initfuncdesc, = s_init.descriptions if isinstance(initfuncdesc, FunctionDesc): initmethdesc = bookkeeper.getmethoddesc(initfuncdesc, classdef, @@ -782,8 +800,8 @@ desc.selfclassdef, desc.name, commonflags) - del descs[desc] - descs[newdesc] = None + descs.remove(desc) + descs.add(newdesc) # --- case 1 --- groups = {} @@ -798,7 +816,7 @@ for desc2 in group: cdef2 = desc2.selfclassdef if cdef1 is not cdef2 and cdef1.issubclass(cdef2): - del descs[desc1] + descs.remove(desc1) break simplify_desc_set = staticmethod(simplify_desc_set) Modified: pypy/branch/jitypes2/pypy/annotation/listdef.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/listdef.py (original) +++ pypy/branch/jitypes2/pypy/annotation/listdef.py Fri Dec 3 11:04:29 2010 @@ -6,11 +6,16 @@ class TooLateForChange(Exception): pass +class ListChangeUnallowed(Exception): + pass + class ListItem(object): mutated = False # True for lists mutated after creation resized = False # True for lists resized after creation range_step = None # the step -- only for lists only created by a range() dont_change_any_more = False # set to True when too late for changes + immutable = False # for getattr out of _immutable_fields_ = ['attr[*]'] + must_not_resize = False # make_sure_not_resized() # what to do if range_step is different in merge. # - if one is a list (range_step is None), unify to a list. @@ -26,7 +31,6 @@ self.bookkeeper = bookkeeper self.itemof = {} # set of all ListDefs using this ListItem self.read_locations = {} - self.dont_resize = False if bookkeeper is None: self.dont_change_any_more = True @@ -34,12 +38,15 @@ if not self.mutated: if self.dont_change_any_more: raise TooLateForChange + self.immutable = False self.mutated = True def resize(self): if not self.resized: - if self.dont_change_any_more or self.dont_resize: + if self.dont_change_any_more: raise TooLateForChange + if self.must_not_resize: + raise ListChangeUnallowed("resizing list") self.resized = True def setrangestep(self, step): @@ -63,11 +70,13 @@ # things more general self, other = other, self - if other.dont_resize: - if self.resized: - raise TooLateForChange() - self.dont_resize = True - if other.mutated: self.mutate() + self.immutable &= other.immutable + if other.must_not_resize: + if self.resized: + raise ListChangeUnallowed("list merge with a resized") + self.must_not_resize = True + if other.mutated: + self.mutate() if other.resized: self.resize() if other.range_step != self.range_step: @@ -176,9 +185,11 @@ self.listitem.generalize(s_value) def __repr__(self): - return '<[%r]%s%s>' % (self.listitem.s_value, + return '<[%r]%s%s%s%s>' % (self.listitem.s_value, self.listitem.mutated and 'm' or '', - self.listitem.resized and 'r' or '') + self.listitem.resized and 'r' or '', + self.listitem.immutable and 'I' or '', + self.listitem.must_not_resize and '!R' or '') def mutate(self): self.listitem.mutate() @@ -189,13 +200,20 @@ def never_resize(self): if self.listitem.resized: - raise TooLateForChange() - self.listitem.dont_resize = True + raise ListChangeUnallowed("list already resized") + self.listitem.must_not_resize = True - def never_mutate(self): - if self.listitem.resized or self.listitem.mutated: - raise TooLateForChange() - self.listitem.dont_change_any_more = True + def mark_as_immutable(self): + # Sets the 'immutable' flag. Note that unlike "never resized", + # the immutable flag is only a hint. It is cleared again e.g. + # when we merge with a "normal" list that doesn't have it. It + # is thus expected to live only shortly, mostly for the case + # of writing 'x.list[n]'. + self.never_resize() + if not self.listitem.mutated: + self.listitem.immutable = True + #else: it's fine, don't set immutable=True at all (see + # test_can_merge_immutable_list_with_regular_list) MOST_GENERAL_LISTDEF = ListDef(None, SomeObject()) Modified: pypy/branch/jitypes2/pypy/annotation/model.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/model.py (original) +++ pypy/branch/jitypes2/pypy/annotation/model.py Fri Dec 3 11:04:29 2010 @@ -356,8 +356,8 @@ immutable = True def __init__(self, descriptions, can_be_None=False, subset_of=None): - # descriptions is a set of Desc instances. - descriptions = dict.fromkeys(descriptions) + # descriptions is a set of Desc instances + descriptions = set(descriptions) self.descriptions = descriptions self.can_be_None = can_be_None self.subset_of = subset_of @@ -379,6 +379,9 @@ if desc.pyobj is not None: self.const = desc.pyobj + def any_description(self): + return iter(self.descriptions).next() + def getKind(self): "Return the common Desc class of all descriptions in this PBC." kinds = {} Modified: pypy/branch/jitypes2/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/specialize.py (original) +++ pypy/branch/jitypes2/pypy/annotation/specialize.py Fri Dec 3 11:04:29 2010 @@ -345,7 +345,8 @@ key.append(s.const) elif isinstance(s, SomePBC) and len(s.descriptions) == 1: # for test_specialize_arg_bound_method - key.append(s.descriptions.keys()[0]) + desc, = s.descriptions + key.append(desc) else: raise Exception("specialize:arg(%d): argument not constant: %r" % (i, s)) Modified: pypy/branch/jitypes2/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/jitypes2/pypy/annotation/test/test_annrpython.py Fri Dec 3 11:04:29 2010 @@ -10,7 +10,7 @@ from pypy.translator.translator import graphof as tgraphof from pypy.annotation import policy from pypy.annotation import specialize -from pypy.annotation.listdef import ListDef, TooLateForChange +from pypy.annotation.listdef import ListDef, ListChangeUnallowed from pypy.annotation.dictdef import DictDef from pypy.objspace.flow.model import * from pypy.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong @@ -1010,7 +1010,7 @@ bookkeeper = a.bookkeeper def getmdesc(bmeth): - return bookkeeper.immutablevalue(bmeth).descriptions.keys()[0] + return bookkeeper.immutablevalue(bmeth).any_description() mdescA_m = getmdesc(A().m) mdescC_m = getmdesc(C().m) @@ -2862,7 +2862,7 @@ assert s.items[0].flags == {'access_directly': True} assert isinstance(s.items[1], annmodel.SomePBC) assert len(s.items[1].descriptions) == 1 - assert s.items[1].descriptions.keys()[0].flags == {'access_directly': + assert s.items[1].any_description().flags == {'access_directly': True} assert isinstance(s.items[2], annmodel.SomeInstance) assert s.items[2].flags == {'access_directly': True} @@ -3206,7 +3206,7 @@ l.append(4) a = self.RPythonAnnotator() - py.test.raises(TooLateForChange, a.build_types, g, []) + py.test.raises(ListChangeUnallowed, a.build_types, g, []) assert called def test_listitem_no_mutating2(self): @@ -3229,7 +3229,7 @@ a = self.RPythonAnnotator() a.translator.config.translation.list_comprehension_operations = True - py.test.raises(TooLateForChange, a.build_types, fn, [int]) + py.test.raises(ListChangeUnallowed, a.build_types, fn, [int]) def test_listitem_never_resize(self): from pypy.rlib.debug import check_annotation @@ -3243,7 +3243,7 @@ check_annotation(l, checker) a = self.RPythonAnnotator() - py.test.raises(TooLateForChange, a.build_types, f, []) + py.test.raises(ListChangeUnallowed, a.build_types, f, []) def test_len_of_empty_list(self): @@ -3357,6 +3357,87 @@ # not a constant: both __enter__ and __exit__ have been annotated assert not s.is_constant() + def test_make_sure_not_resized(self): + from pypy.rlib.debug import make_sure_not_resized + + def pycode(consts): + make_sure_not_resized(consts) + def build1(): + return pycode(consts=[1]) + def build2(): + return pycode(consts=[0]) + def fn(): + build1() + build2() + + a = self.RPythonAnnotator() + a.translator.config.translation.list_comprehension_operations = True + a.build_types(fn, []) + # assert did not raise ListChangeUnallowed + + def test_return_immutable_list(self): + class A: + _immutable_fields_ = 'lst[*]' + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + return a.lst + + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert s.listdef.listitem.immutable + + def test_immutable_list_is_actually_resized(self): + class A: + _immutable_fields_ = 'lst[*]' + def f(n): + a = A() + l1 = [n] + l1.append(n+1) + a.lst = l1 + return a.lst + + a = self.RPythonAnnotator() + py.test.raises(ListChangeUnallowed, a.build_types, f, [int]) + + def test_can_merge_immutable_list_with_regular_list(self): + class A: + _immutable_fields_ = 'lst[*]' + def foo(lst): + pass + + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + if n > 0: + foo(a.lst) + else: + lst = [0] + lst[0] = n + foo(lst) + + a = self.RPythonAnnotator() + a.build_types(f, [int]) + + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + if n > 0: + lst = [0] + lst[0] = n + foo(lst) + else: + foo(a.lst) + + a = self.RPythonAnnotator() + a.build_types(f, [int]) + def g(n): return [0,1,2,n] Modified: pypy/branch/jitypes2/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/unaryop.py (original) +++ pypy/branch/jitypes2/pypy/annotation/unaryop.py Fri Dec 3 11:04:29 2010 @@ -615,6 +615,9 @@ if basedef.classdesc.all_enforced_attrs is not None: if attr in basedef.classdesc.all_enforced_attrs: raise HarmlesslyBlocked("get enforced attr") + elif isinstance(s_result, SomeList): + s_result = ins.classdef.classdesc.maybe_return_immutable_list( + attr, s_result) return s_result return SomeObject() getattr.can_only_throw = [] Modified: pypy/branch/jitypes2/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/jitypes2/pypy/config/pypyoption.py (original) +++ pypy/branch/jitypes2/pypy/config/pypyoption.py Fri Dec 3 11:04:29 2010 @@ -31,7 +31,7 @@ "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", - "_bisect", "_ffi"] + "_bisect", "binascii", "_ffi"] )) translation_modules = default_modules.copy() @@ -161,7 +161,6 @@ suggests=[("objspace.allworkingmodules", False)]), BoolOption("geninterp", "specify whether geninterp should be used", - cmdline=None, default=True), BoolOption("logbytecodes", Modified: pypy/branch/jitypes2/pypy/config/translationoption.py ============================================================================== --- pypy/branch/jitypes2/pypy/config/translationoption.py (original) +++ pypy/branch/jitypes2/pypy/config/translationoption.py Fri Dec 3 11:04:29 2010 @@ -3,6 +3,7 @@ from pypy.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption, FloatOption from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config from pypy.config.config import ConfigError +from pypy.config.support import detect_number_of_processors DEFL_INLINE_THRESHOLD = 32.4 # just enough to inline add__Int_Int() # and just small enough to prevend inlining of some rlist functions. @@ -113,10 +114,6 @@ ChoiceOption("jit_backend", "choose the backend for the JIT", ["auto", "x86", "x86-without-sse2", "llvm"], default="auto", cmdline="--jit-backend"), - ChoiceOption("jit_debug", "the amount of debugging dumps made by the JIT", - ["off", "profile", "steps", "detailed"], - default="off", - cmdline="--jit-debug"), ChoiceOption("jit_profiler", "integrate profiler support into the JIT", ["off", "oprofile"], default="off"), @@ -171,7 +168,7 @@ default=False, negation=False), IntOption("make_jobs", "Specify -j argument to make for compilation" " (C backend only)", - cmdline="--make-jobs", default=1), + cmdline="--make-jobs", default=detect_number_of_processors()), # Flags of the TranslationContext: BoolOption("simplifying", "Simplify flow graphs", default=True), Modified: pypy/branch/jitypes2/pypy/conftest.py ============================================================================== --- pypy/branch/jitypes2/pypy/conftest.py (original) +++ pypy/branch/jitypes2/pypy/conftest.py Fri Dec 3 11:04:29 2010 @@ -336,13 +336,15 @@ self.runtest_finish() def runtest_open(self): - leakfinder.start_tracking_allocations() + if not getattr(self.obj, 'dont_track_allocations', False): + leakfinder.start_tracking_allocations() def runtest_perform(self): super(PyPyTestFunction, self).runtest() def runtest_close(self): - if leakfinder.TRACK_ALLOCATIONS: + if (not getattr(self.obj, 'dont_track_allocations', False) + and leakfinder.TRACK_ALLOCATIONS): self._pypytest_leaks = leakfinder.stop_tracking_allocations(False) else: # stop_tracking_allocations() already called self._pypytest_leaks = None Modified: pypy/branch/jitypes2/pypy/doc/cpython_differences.txt ============================================================================== --- pypy/branch/jitypes2/pypy/doc/cpython_differences.txt (original) +++ pypy/branch/jitypes2/pypy/doc/cpython_differences.txt Fri Dec 3 11:04:29 2010 @@ -20,16 +20,21 @@ __builtin__ `__pypy__`_ + _ast + _bisect _codecs _lsprof `_minimal_curses`_ _random `_rawffi`_ + _ssl _socket _sre _weakref + array bz2 cStringIO + `cpyext`_ crypt errno exceptions @@ -72,7 +77,7 @@ * Supported by being rewritten in pure Python (possibly using ``ctypes``): see the `lib_pypy/`_ directory. Examples of modules that we - support this way: ``ctypes``, ``array``, ``cPickle``, + support this way: ``ctypes``, ``cPickle``, ``cStringIO``, ``cmath``, ``dbm`` (?), ``datetime``, ``binascii``... Note that some modules are both in there and in the list above; by default, the built-in module is used (but can be disabled @@ -80,11 +85,13 @@ The extension modules (i.e. modules written in C, in the standard CPython) that are neither mentioned above nor in `lib_pypy/`_ are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) .. the nonstandard modules are listed below... .. _`__pypy__`: __pypy__-module.html .. _`_rawffi`: ctypes-implementation.html .. _`_minimal_curses`: config/objspace.usemodules._minimal_curses.html +.. _`cpyext`: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html .. _Stackless: stackless.html @@ -129,12 +136,10 @@ .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html -The built-in function ``id()`` returns numbers that are not addresses -for most of PyPy's garbage collectors. -This is most visible in the default repr: a typical PyPy object can -pretend to be located ``at 0x00000009``. This is just its ``id()``, not -its real address (because an object can move around in some GCs). Calling -``id`` a lot can lead to performance problem. +Using the default GC called ``minimark``, the built-in function ``id()`` +works like it does in CPython. With other GCs it returns numbers that +are not real addresses (because an object can move around several times) +and calling it a lot can lead to performance problem. Note that if you have a long chain of objects, each with a reference to the next one, and each with a __del__, PyPy's GC will perform badly. On Modified: pypy/branch/jitypes2/pypy/doc/faq.txt ============================================================================== --- pypy/branch/jitypes2/pypy/doc/faq.txt (original) +++ pypy/branch/jitypes2/pypy/doc/faq.txt Fri Dec 3 11:04:29 2010 @@ -47,7 +47,7 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has been a part of 1.3 release, but support is still +change) on trunk. It has been a part of 1.4 release, but support is still in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules Modified: pypy/branch/jitypes2/pypy/doc/index.txt ============================================================================== --- pypy/branch/jitypes2/pypy/doc/index.txt (original) +++ pypy/branch/jitypes2/pypy/doc/index.txt Fri Dec 3 11:04:29 2010 @@ -8,7 +8,7 @@ Getting into PyPy ... ============================================= -* `Release 1.3`_: the latest official release +* `Release 1.4`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -56,4 +56,4 @@ .. _`Documentation`: docindex.html .. _`Getting Started`: getting-started.html .. _papers: extradoc.html -.. _`Release 1.3`: http://pypy.org/download.html +.. _`Release 1.4`: http://pypy.org/download.html Modified: pypy/branch/jitypes2/pypy/doc/release-1.4.0.txt ============================================================================== --- pypy/branch/jitypes2/pypy/doc/release-1.4.0.txt (original) +++ pypy/branch/jitypes2/pypy/doc/release-1.4.0.txt Fri Dec 3 11:04:29 2010 @@ -2,27 +2,58 @@ PyPy 1.4: Ouroboros in practice =============================== -Hello. - We're pleased to announce the 1.4 release of PyPy. This is a major breakthrough in our long journey, as PyPy 1.4 is the first PyPy release that can translate -itself faster than CPython. Starting today, we plan to start using PyPy for our -own development. +itself faster than CPython. Starting today, we are using PyPy more for +our every-day development. So may you :) You can download it here: + + http://pypy.org/download.html + +What is PyPy +============ + +PyPy is a very compliant Python interpreter, almost a drop-in replacement +for CPython. It's fast (`pypy 1.4 and cpython 2.6`_ comparison) -Among other features, this release includes numerous performance improvements +Among its new features, this release includes numerous performance improvements (which made fast self-hosting possible), a 64-bit JIT backend, as well as serious stabilization. As of now, we can consider the 32-bit and 64-bit -linux versions of PyPy stable enoughto run in production. +linux versions of PyPy stable enough to run `in production`_. + +Numerous speed achievements are described on `our blog`_. Normalized speed +charts comparing `pypy 1.4 and pypy 1.3`_ as well as `pypy 1.4 and cpython 2.6`_ +are available on benchmark website. For the impatient: yes, we got a lot faster! More highlights =============== -* Virtualenv support: now PyPy is fully compatible with virtualenv_: note that +* PyPy's built-in Just-in-Time compiler is fully transparent and + automatically generated; it now also has very reasonable memory + requirements. The total memory used by a very complex and + long-running process (translating PyPy itself) is within 1.5x to + at most 2x the memory needed by CPython, for a speed-up of 2x. + +* More compact instances. All instances are as compact as if + they had ``__slots__``. This can give programs a big gain in + memory. (In the example of translation above, we already have + carefully placed ``__slots__``, so there is no extra win.) + +* `Virtualenv support`_: now PyPy is fully compatible with virtualenv_: note that to use it, you need a recent version of virtualenv (>= 1.5). * Faster (and JITted) regular expressions - huge boost in speeding up - sre module. + the `re` module. -* Faster (and JITted) calls to functions like map(). +* Other speed improvements, like JITted calls to functions like map(). .. _virtualenv: http://pypi.python.org/pypi/virtualenv +.. _`Virtualenv support`: http://morepypy.blogspot.com/2010/08/using-virtualenv-with-pypy.html +.. _`in production`: http://morepypy.blogspot.com/2010/11/running-large-radio-telescope-software.html +.. _`our blog`: http://morepypy.blogspot.com +.. _`pypy 1.4 and pypy 1.3`: http://speed.pypy.org/comparison/?exe=1%2B41,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=1%2B41&chart=normal+bars +.. _`pypy 1.4 and cpython 2.6`: http://speed.pypy.org/comparison/?exe=2%2B35,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=2%2B35&chart=normal+bars + +Cheers, + +Carl Friedrich Bolz, Antonio Cuni, Maciej Fijalkowski, +Amaury Forgeot d'Arc, Armin Rigo and the PyPy team Modified: pypy/branch/jitypes2/pypy/doc/sprint-reports.txt ============================================================================== --- pypy/branch/jitypes2/pypy/doc/sprint-reports.txt (original) +++ pypy/branch/jitypes2/pypy/doc/sprint-reports.txt Fri Dec 3 11:04:29 2010 @@ -30,6 +30,17 @@ * `D?sseldorf (October 2006)`_ * `Leysin (January 2007)`_ * `Hildesheim (Feb 2007)`_ (also `EU report writing sprint`_) + * `G?teborg (November 2007)`_ + * `Leysin (January 2008)`_ + * `Berlin (May 2008)`_ + * `Vilnius after EuroPython (July 2008)`_ + * `D?sseldorf (August 2008)`_ + * `Wroclaw (February 2009)`_ + * `Leysin (April 2009)`_ + * `G?teborg (August 2009)`_ + * `D?sseldorf (November 2009)`_ + * `CERN (July 2010)`_ + * `D?sseldorf (October 2010)`_ .. _Hildesheim (Feb 2003): http://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html .. _Gothenburg (May 2003): http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt @@ -55,3 +66,15 @@ .. _Hildesheim (Feb 2007): http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/sprint-report.txt .. _`EU report writing sprint`: http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/eu-report-sprint-report.txt .. _`PyCon/Dallas (Feb 2006)`: http://codespeak.net/pypy/extradoc/sprintinfo/pycon06/sprint-report.txt + .. _`G?teborg (November 2007)`: http://morepypy.blogspot.com/2007_11_01_archive.html + .. _`Leysin (January 2008)`: http://morepypy.blogspot.com/2008/01/leysin-winter-sport-sprint-started.html + .. _`Berlin (May 2008)`: http://morepypy.blogspot.com/2008_05_01_archive.html + .. _`Vilnius after EuroPython (July 2008)`: http://morepypy.blogspot.com/2008/07/europython-2008-pypy-talks-and-sprint.html + .. _`D?sseldorf (August 2008)`: http://morepypy.blogspot.com/2008_10_01_archive.html + .. _`Wroclaw (February 2009)`: http://morepypy.blogspot.com/2009/02/wroclaw-2009-sprint-progress-report.html + .. _`Leysin (April 2009)`: http://morepypy.blogspot.com/2009/04/leysin-sprint-report.html + .. _`G?teborg (August 2009)`: http://morepypy.blogspot.com/2009/08/gothenburg-jit-sprint-report.html + .. _`D?sseldorf (November 2009)`: http://morepypy.blogspot.com/2009/11/dusseldorf-sprint-report.html + .. _`CERN (July 2010)`: http://morepypy.blogspot.com/2010/07/cern-sprint-report-wrapping-c-libraries.html + .. _`D?sseldorf (October 2010)`: http://morepypy.blogspot.com/2010/10/dusseldorf-sprint-report-2010.html + Modified: pypy/branch/jitypes2/pypy/doc/statistic/release_dates.dat ============================================================================== --- pypy/branch/jitypes2/pypy/doc/statistic/release_dates.dat (original) +++ pypy/branch/jitypes2/pypy/doc/statistic/release_dates.dat Fri Dec 3 11:04:29 2010 @@ -7,3 +7,7 @@ 2006-06-25,"PyPy 0.9" 2007-02-17,"PyPy 0.99" 2007-03-27,"PyPy 1.0" +2009-04-28,"PyPy 1.1" +2010-03-12,"PyPy 1.2" +2010-06-26,"PyPy 1.3" +2010-11-26,"PyPy 1.4" Modified: pypy/branch/jitypes2/pypy/doc/statistic/sprint_dates.dat ============================================================================== --- pypy/branch/jitypes2/pypy/doc/statistic/sprint_dates.dat (original) +++ pypy/branch/jitypes2/pypy/doc/statistic/sprint_dates.dat Fri Dec 3 11:04:29 2010 @@ -1,26 +1,39 @@ PyPy sprints location, begin, end "Hildesheim",2003-02-17,2003-02-23 -"Gothenburg",2003-05-24,2003-05-31 -"LovainLaNeuve",2003-06-21,2003-06-24 +"G??teborg",2003-05-24,2003-05-31 +"Louvain-la-Neuve",2003-06-21,2003-06-24 "Berlin",2003-09-29,2003-10-04 "Amsterdam",2003-12-14,2003-12-21 -"Europython/Gothenburg",2004-06-01,2004-06-07 +"Europython/G??teborg",2004-06-01,2004-06-07 "Vilnius",2004-11-15,2004-11-23 "Leysin",2005-01-22,2005-01-29 "PyCon/Washington",2005-03-19,2005-03-22 -"Europython/Gothenburg",2005-07-01,2005-07-07 +"Europython/G??teborg",2005-07-01,2005-07-07 "Hildesheim",2005-07-25,2005-07-31 "Heidelberg",2005-08-22,2005-08-29 "Paris",2005-10-10,2005-10-16 -"Gothenburg",2005-12-05,2005-12-11 +"G??teborg",2005-12-05,2005-12-11 "Mallorca",2006-01-23,2006-01-29 "Pycon/Dallas",2006-02-27,2006-03-02 "Louvain-la-Neuve",2006-03-06,2006-03-10 "Japan",2006-04-23,2006-04-29 -"Duesseldorf",2006-06-02,2006-06-09 +"D??sseldorf",2006-06-02,2006-06-09 "Europython/Genf",2006-07-06,2006-07-09 "Limerick",2006-08-21,2006-08-27 -"Duesseldorf",2006-10-30,2006-11-05 +"D??sseldorf",2006-10-30,2006-11-05 "Leysin",2007-01-08,2007-01-14 "Hildesheim",2007-03-01,2007-03-05 +"Hildesheim",2007-03-18,2007-03-23 +"Bern",2007-10-22,2007-10-26 +"G??teborg",2007-11-19,2007-11-25 +"Leysin",2008-01-12,2008-01-19 +"Berlin",2008-05-17,2008-05-22 +"EuroPython/Vilnius",2008-07-10,2008-07-12 +"D??sseldorf",2008-08-05,2008-08-13 +"Wroclaw",2009-02-07,2009-02-14 +"Leysin",2009-04-14,2009-04-21 +"G??teborg",2009-08-18,2009-08-25 +"D??sseldorf",2009-11-06,2009-11-13 +"CERN",2010-07-05,2010-07-09 +"D??sseldorf",2010-10-25,2010-10-31 Modified: pypy/branch/jitypes2/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/baseobjspace.py Fri Dec 3 11:04:29 2010 @@ -147,7 +147,7 @@ __already_enqueued_for_destruction = False - def _enqueue_for_destruction(self, space): + def _enqueue_for_destruction(self, space, call_user_del=True): """Put the object in the destructor queue of the space. At a later, safe point in time, UserDelAction will use space.userdel() to call the object's app-level __del__ method. @@ -160,7 +160,8 @@ return self.__already_enqueued_for_destruction = True self.clear_all_weakrefs() - space.user_del_action.register_dying_object(self) + if call_user_del: + space.user_del_action.register_dying_object(self) def _call_builtin_destructor(self): pass # method overridden in typedef.py Modified: pypy/branch/jitypes2/pypy/interpreter/function.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/function.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/function.py Fri Dec 3 11:04:29 2010 @@ -612,11 +612,9 @@ self.w_func_dict = func.w_func_dict self.w_module = func.w_module - def descr_builtinfunction__new__(space, w_subtype, w_func): - func = space.interp_w(Function, w_func) - bltin = space.allocate_instance(BuiltinFunction, w_subtype) - BuiltinFunction.__init__(bltin, func) - return space.wrap(bltin) + def descr_builtinfunction__new__(space, w_subtype): + raise OperationError(space.w_TypeError, + space.wrap("cannot create 'builtin_function' instances")) def descr_function_repr(self): return self.space.wrap('' % (self.name,)) Modified: pypy/branch/jitypes2/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/gateway.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/gateway.py Fri Dec 3 11:04:29 2010 @@ -1083,7 +1083,7 @@ # these decorators are known to return the same function # object, we may ignore them assert '\n' in source - source = source[source.find('\n') + 1:] + source = source[source.find('\n') + 1:].lstrip() assert source.startswith("def "), "can only transform functions" source = source[4:] p = source.find('(') Modified: pypy/branch/jitypes2/pypy/interpreter/generator.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/generator.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/generator.py Fri Dec 3 11:04:29 2010 @@ -10,7 +10,7 @@ def __init__(self, frame): self.space = frame.space - self.frame = frame + self.frame = frame # turned into None when frame_finished_execution self.running = False def descr__reduce__(self, space): @@ -19,9 +19,13 @@ mod = space.interp_w(MixedModule, w_mod) new_inst = mod.get('generator_new') w = space.wrap + if self.frame: + w_frame = w(self.frame) + else: + w_frame = space.w_None tup = [ - w(self.frame), + w_frame, w(self.running), ] @@ -41,7 +45,8 @@ if self.running: raise OperationError(space.w_ValueError, space.wrap('generator already executing')) - if self.frame.frame_finished_execution: + frame = self.frame + if frame is None: # xxx a bit ad-hoc, but we don't want to go inside # execute_generator_frame() if the frame is actually finished if operr is None: @@ -49,7 +54,7 @@ raise operr # XXX it's not clear that last_instr should be promoted at all # but as long as it is necessary for call_assembler, let's do it early - last_instr = jit.hint(self.frame.last_instr, promote=True) + last_instr = jit.hint(frame.last_instr, promote=True) if last_instr == -1: if w_arg and not space.is_w(w_arg, space.w_None): msg = "can't send non-None value to a just-started generator" @@ -60,18 +65,19 @@ self.running = True try: try: - w_result = self.frame.execute_generator_frame(w_arg, operr) + w_result = frame.execute_generator_frame(w_arg, operr) except OperationError: # errors finish a frame - self.frame.frame_finished_execution = True + self.frame = None raise # if the frame is now marked as finished, it was RETURNed from - if self.frame.frame_finished_execution: + if frame.frame_finished_execution: + self.frame = None raise OperationError(space.w_StopIteration, space.w_None) else: return w_result # YIELDed finally: - self.frame.f_backref = jit.vref_None + frame.f_backref = jit.vref_None self.running = False def descr_throw(self, w_type, w_val=None, w_tb=None): @@ -115,7 +121,7 @@ raise OperationError(space.w_RuntimeError, space.wrap(msg)) def descr_gi_frame(space, self): - if not self.frame.frame_finished_execution: + if self.frame is not None and not self.frame.frame_finished_execution: return self.frame else: return space.w_None @@ -125,15 +131,17 @@ applevel __del__, which is called at a safe point after the interp-level __del__ enqueued the object for destruction """ - # Only bother raising an exception if the frame is still not - # finished and finally or except blocks are present. - if not self.frame.frame_finished_execution: + self.descr_close() + + def __del__(self): + # Only bother enqueuing self to raise an exception if the frame is + # still not finished and finally or except blocks are present. + must_call_close = False + if self.frame is not None: block = self.frame.lastblock while block is not None: if not isinstance(block, LoopBlock): - self.descr_close() - return + must_call_close = True + break block = block.previous - - def __del__(self): - self._enqueue_for_destruction(self.space) + self._enqueue_for_destruction(self.space, must_call_close) Modified: pypy/branch/jitypes2/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/pycode.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/pycode.py Fri Dec 3 11:04:29 2010 @@ -15,7 +15,7 @@ CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask -from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified +from pypy.rlib.debug import make_sure_not_resized from pypy.rlib import jit from pypy.rlib.objectmodel import compute_hash from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT @@ -69,7 +69,7 @@ self.co_stacksize = stacksize self.co_flags = flags self.co_code = code - self.co_consts_w = make_sure_not_modified(consts) + self.co_consts_w = consts self.co_names_w = [space.new_interned_str(aname) for aname in names] self.co_varnames = varnames self.co_freevars = freevars @@ -269,7 +269,7 @@ dis.dis(co) def fget_co_consts(space, self): - return space.newtuple(self.co_consts_w[:]) + return space.newtuple(self.co_consts_w) def fget_co_names(space, self): return space.newtuple(self.co_names_w) @@ -383,7 +383,7 @@ w(self.co_stacksize), w(self.co_flags), w(self.co_code), - space.newtuple(self.co_consts_w[:]), + space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), Modified: pypy/branch/jitypes2/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/test/test_gateway.py Fri Dec 3 11:04:29 2010 @@ -578,6 +578,11 @@ w_res = space.call_args(w_g, args) assert space.eq_w(w_res, space.wrap((-1, 0))) +class AppTestPyTestMark: + @py.test.mark.unlikely_to_exist + def test_anything(self): + pass + class TestPassThroughArguments: Modified: pypy/branch/jitypes2/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/detect_cpu.py Fri Dec 3 11:04:29 2010 @@ -31,7 +31,8 @@ 'i86pc': 'x86', # Solaris/Intel 'x86': 'x86', # Apple 'Power Macintosh': 'ppc', - 'x86_64': 'x86', + 'x86_64': 'x86', + 'amd64': 'x86' # freebsd }[mach] except KeyError: return mach Modified: pypy/branch/jitypes2/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/llgraph/llimpl.py Fri Dec 3 11:04:29 2010 @@ -4,6 +4,7 @@ when executing on top of the llinterpreter. """ +import weakref from pypy.objspace.flow.model import Variable, Constant from pypy.annotation import model as annmodel from pypy.jit.metainterp.history import (ConstInt, ConstPtr, @@ -161,6 +162,8 @@ # ____________________________________________________________ class CompiledLoop(object): + has_been_freed = False + def __init__(self): self.inputargs = [] self.operations = [] @@ -285,6 +288,11 @@ del _variables[:] return _to_opaque(CompiledLoop()) +def mark_as_free(loop): + loop = _from_opaque(loop) + assert not loop.has_been_freed + loop.has_been_freed = True + def compile_start_int_var(loop): return compile_start_ref_var(loop, lltype.Signed) @@ -317,7 +325,7 @@ raise ValueError("CALL_ASSEMBLER not supported") loop = _from_opaque(loop) op = loop.operations[-1] - op.descr = descr + op.descr = weakref.ref(descr) def compile_add_var(loop, intvar): loop = _from_opaque(loop) @@ -429,6 +437,7 @@ verbose = True self.opindex = 0 while True: + assert not self.loop.has_been_freed op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): @@ -440,7 +449,10 @@ _stats.exec_conditional_jumps += 1 if op.jump_target is not None: # a patched guard, pointing to further code - args = [self.getenv(v) for v in op.fail_args if v] + if op.fail_args: + args = [self.getenv(v) for v in op.fail_args if v] + else: + args = [] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target @@ -571,7 +583,12 @@ def op_debug_merge_point(self, _, value, recdepth): from pypy.jit.metainterp.warmspot import get_stats loc = ConstPtr(value)._get_str() - get_stats().add_merge_point_location(loc) + try: + stats = get_stats() + except AttributeError: + pass + else: + stats.add_merge_point_location(loc) def op_guard_true(self, _, value): if not value: @@ -839,14 +856,22 @@ finally: self._may_force = -1 - def op_call_assembler(self, loop_token, *args): + def op_call_assembler(self, wref_loop_token, *args): + if we_are_translated(): + raise ValueError("CALL_ASSEMBLER not supported") + return self._do_call_assembler(wref_loop_token, *args) + + def _do_call_assembler(self, wref_loop_token, *args): global _last_exception + loop_token = wref_loop_token() + assert loop_token, "CALL_ASSEMBLER to a target that already died" + ctl = loop_token.compiled_loop_token + if hasattr(ctl, 'redirected'): + return self._do_call_assembler(ctl.redirected, *args) assert not self._forced - loop_token = self.cpu._redirected_call_assembler.get(loop_token, - loop_token) self._may_force = self.opindex try: - inpargs = _from_opaque(loop_token._llgraph_compiled_version).inputargs + inpargs = _from_opaque(ctl.compiled_version).inputargs for i, inparg in enumerate(inpargs): TYPE = inparg.concretetype if TYPE is lltype.Signed: @@ -1539,10 +1564,13 @@ do_setfield_gc_int(vable, fielddescr.ofs, 0) def redirect_call_assembler(cpu, oldlooptoken, newlooptoken): - OLD = _from_opaque(oldlooptoken._llgraph_compiled_version).getargtypes() - NEW = _from_opaque(newlooptoken._llgraph_compiled_version).getargtypes() + oldclt = oldlooptoken.compiled_loop_token + newclt = newlooptoken.compiled_loop_token + OLD = _from_opaque(oldclt.compiled_version).getargtypes() + NEW = _from_opaque(newclt.compiled_version).getargtypes() assert OLD == NEW - cpu._redirected_call_assembler[oldlooptoken] = newlooptoken + assert not hasattr(oldclt, 'redirected') + oldclt.redirected = weakref.ref(newlooptoken) # ____________________________________________________________ @@ -1609,6 +1637,7 @@ setannotation(compile_add_fail, annmodel.SomeInteger()) setannotation(compile_add_fail_arg, annmodel.s_None) setannotation(compile_redirect_fail, annmodel.s_None) +setannotation(mark_as_free, annmodel.s_None) setannotation(new_frame, s_Frame) setannotation(frame_clear, annmodel.s_None) Modified: pypy/branch/jitypes2/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/llgraph/runner.py Fri Dec 3 11:04:29 2010 @@ -102,7 +102,6 @@ llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] self._descrs = {} - self._redirected_call_assembler = {} def _freeze_(self): assert self.translate_support_code @@ -118,22 +117,34 @@ self._descrs[key] = descr return descr - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): c = llimpl.compile_start() + clt = original_loop_token.compiled_loop_token + clt.loop_and_bridges.append(c) + clt.compiling_a_bridge() self._compile_loop_or_bridge(c, inputargs, operations) old, oldindex = faildescr._compiled_fail llimpl.compile_redirect_fail(old, oldindex, c) - def compile_loop(self, inputargs, operations, loopdescr, log=True): + def compile_loop(self, inputargs, operations, looptoken, log=True): """In a real assembler backend, this should assemble the given list of operations. Here we just generate a similar CompiledLoop instance. The code here is RPython, whereas the code in llimpl is not. """ c = llimpl.compile_start() - loopdescr._llgraph_compiled_version = c + clt = model.CompiledLoopToken(self, looptoken.number) + clt.loop_and_bridges = [c] + clt.compiled_version = c + looptoken.compiled_loop_token = clt self._compile_loop_or_bridge(c, inputargs, operations) + def free_loop_and_bridges(self, compiled_loop_token): + for c in compiled_loop_token.loop_and_bridges: + llimpl.mark_as_free(c) + model.AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) + def _compile_loop_or_bridge(self, c, inputargs, operations): var2index = {} for box in inputargs: @@ -207,7 +218,7 @@ if op.getopnum() == rop.JUMP: targettoken = op.getdescr() assert isinstance(targettoken, history.LoopToken) - compiled_version = targettoken._llgraph_compiled_version + compiled_version = targettoken.compiled_loop_token.compiled_version llimpl.compile_add_jump_target(c, compiled_version) elif op.getopnum() == rop.FINISH: faildescr = op.getdescr() @@ -217,7 +228,7 @@ assert False, "unknown operation" def _execute_token(self, loop_token): - compiled_version = loop_token._llgraph_compiled_version + compiled_version = loop_token.compiled_loop_token.compiled_version frame = llimpl.new_frame(self.is_oo, self) # setup the frame llimpl.frame_clear(frame, compiled_version) Modified: pypy/branch/jitypes2/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/llsupport/gc.py Fri Dec 3 11:04:29 2010 @@ -15,6 +15,7 @@ from pypy.jit.backend.llsupport.descr import GcCache, get_field_descr from pypy.jit.backend.llsupport.descr import GcPtrFieldDescr from pypy.jit.backend.llsupport.descr import get_call_descr +from pypy.rpython.memory.gctransform import asmgcroot # ____________________________________________________________ @@ -35,6 +36,8 @@ return False def has_write_barrier_class(self): return None + def freeing_block(self, start, stop): + pass # ____________________________________________________________ @@ -218,50 +221,120 @@ LOC_EBP_PLUS = 2 LOC_EBP_MINUS = 3 - GCMAP_ARRAY = rffi.CArray(llmemory.Address) - CALLSHAPE_ARRAY = rffi.CArray(rffi.UCHAR) + GCMAP_ARRAY = rffi.CArray(lltype.Signed) + CALLSHAPE_ARRAY_PTR = rffi.CArrayPtr(rffi.UCHAR) def __init__(self): + # '_gcmap' is an array of length '_gcmap_maxlength' of addresses. + # '_gcmap_curlength' tells how full the array really is. + # The addresses are actually grouped in pairs: + # (addr-after-the-CALL-in-assembler, addr-of-the-call-shape). + # '_gcmap_deadentries' counts pairs marked dead (2nd item is NULL). + # '_gcmap_sorted' is True only if we know the array is sorted. self._gcmap = lltype.nullptr(self.GCMAP_ARRAY) self._gcmap_curlength = 0 self._gcmap_maxlength = 0 + self._gcmap_deadentries = 0 + self._gcmap_sorted = True def initialize(self): # hack hack hack. Remove these lines and see MissingRTypeAttribute # when the rtyper tries to annotate these methods only when GC-ing... self.gcmapstart() self.gcmapend() + self.gcmarksorted() def gcmapstart(self): - return llmemory.cast_ptr_to_adr(self._gcmap) + return rffi.cast(llmemory.Address, self._gcmap) def gcmapend(self): addr = self.gcmapstart() if self._gcmap_curlength: - addr += llmemory.sizeof(llmemory.Address)*self._gcmap_curlength + addr += rffi.sizeof(lltype.Signed) * self._gcmap_curlength + if not we_are_translated() and type(addr) is long: + from pypy.rpython.lltypesystem import ll2ctypes + addr = ll2ctypes._lladdress(addr) # XXX workaround return addr + def gcmarksorted(self): + # Called by the GC when it is about to sort [gcmapstart():gcmapend()]. + # Returns the previous sortedness flag -- i.e. returns True if it + # is already sorted, False if sorting is needed. + sorted = self._gcmap_sorted + self._gcmap_sorted = True + return sorted + def put(self, retaddr, callshapeaddr): """'retaddr' is the address just after the CALL. - 'callshapeaddr' is the address returned by encode_callshape().""" + 'callshapeaddr' is the address of the raw 'shape' marker. + Both addresses are actually integers here.""" index = self._gcmap_curlength if index + 2 > self._gcmap_maxlength: - self._enlarge_gcmap() + index = self._enlarge_gcmap() self._gcmap[index] = retaddr self._gcmap[index+1] = callshapeaddr self._gcmap_curlength = index + 2 + self._gcmap_sorted = False + @rgc.no_collect def _enlarge_gcmap(self): - newlength = 250 + self._gcmap_maxlength * 2 - newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', - track_allocation=False) oldgcmap = self._gcmap - for i in range(self._gcmap_curlength): - newgcmap[i] = oldgcmap[i] - self._gcmap = newgcmap - self._gcmap_maxlength = newlength - if oldgcmap: - lltype.free(oldgcmap, flavor='raw', track_allocation=False) + if self._gcmap_deadentries * 3 * 2 > self._gcmap_maxlength: + # More than 1/3rd of the entries are dead. Don't actually + # enlarge the gcmap table, but just clean up the dead entries. + newgcmap = oldgcmap + else: + # Normal path: enlarge the array. + newlength = 250 + (self._gcmap_maxlength // 3) * 4 + newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', + track_allocation=False) + self._gcmap_maxlength = newlength + # + j = 0 + i = 0 + end = self._gcmap_curlength + while i < end: + if oldgcmap[i + 1]: + newgcmap[j] = oldgcmap[i] + newgcmap[j + 1] = oldgcmap[i + 1] + j += 2 + i += 2 + self._gcmap_curlength = j + self._gcmap_deadentries = 0 + if oldgcmap != newgcmap: + self._gcmap = newgcmap + if oldgcmap: + lltype.free(oldgcmap, flavor='raw', track_allocation=False) + return j + + @rgc.no_collect + def freeing_block(self, start, stop): + # if [start:stop] is a raw block of assembler, then look up the + # corresponding gcroot markers, and mark them as freed now in + # self._gcmap by setting the 2nd address of every entry to NULL. + gcmapstart = self.gcmapstart() + gcmapend = self.gcmapend() + if gcmapstart == gcmapend: + return + if not self.gcmarksorted(): + asmgcroot.sort_gcmap(gcmapstart, gcmapend) + # A note about gcmarksorted(): the deletion we do here keeps the + # array sorted. This avoids needing too many sort_gcmap()s. + # Indeed, freeing_block() is typically called many times in a row, + # so it will call sort_gcmap() at most the first time. + startaddr = rffi.cast(llmemory.Address, start) + stopaddr = rffi.cast(llmemory.Address, stop) + item = asmgcroot.binary_search(gcmapstart, gcmapend, startaddr) + # 'item' points to one of the entries. Because the whole array + # is sorted, we know that it points either to the first entry we + # want to kill, or to the previous entry. + if item.address[0] < startaddr: + item += asmgcroot.arrayitemsize # go forward one entry + assert item == gcmapend or item.address[0] >= startaddr + while item != gcmapend and item.address[0] < stopaddr: + item.address[1] = llmemory.NULL + self._gcmap_deadentries += 1 + item += asmgcroot.arrayitemsize def get_basic_shape(self, is_64_bit=False): # XXX: Should this code even really know about stack frame layout of @@ -304,17 +377,15 @@ assert reg_index > 0 shape.append(chr(self.LOC_REG | (reg_index << 2))) - def compress_callshape(self, shape): + def compress_callshape(self, shape, datablockwrapper): # Similar to compress_callshape() in trackgcroot.py. - # XXX so far, we always allocate a new small array (we could regroup - # them inside bigger arrays) and we never try to share them. + # Returns an address to raw memory (as an integer). length = len(shape) - compressed = lltype.malloc(self.CALLSHAPE_ARRAY, length, - flavor='raw', - track_allocation=False) # memory leak + rawaddr = datablockwrapper.malloc_aligned(length, 1) + p = rffi.cast(self.CALLSHAPE_ARRAY_PTR, rawaddr) for i in range(length): - compressed[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) - return llmemory.cast_ptr_to_adr(compressed) + p[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) + return rawaddr class WriteBarrierDescr(AbstractDescr): @@ -379,6 +450,7 @@ 'layoutbuilder': self.layoutbuilder, 'gcmapstart': lambda: gcrootmap.gcmapstart(), 'gcmapend': lambda: gcrootmap.gcmapend(), + 'gcmarksorted': lambda: gcrootmap.gcmarksorted(), } self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc @@ -641,6 +713,9 @@ def has_write_barrier_class(self): return WriteBarrierDescr + def freeing_block(self, start, stop): + self.gcrootmap.freeing_block(start, stop) + # ____________________________________________________________ def get_ll_description(gcdescr, translator=None, rtyper=None): Modified: pypy/branch/jitypes2/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/llsupport/llmodel.py Fri Dec 3 11:04:29 2010 @@ -18,6 +18,7 @@ from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr from pypy.jit.backend.llsupport.ffisupport import get_call_descr_dynamic +from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -52,6 +53,7 @@ else: self._setup_exception_handling_untranslated() self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) + self.asmmemmgr = AsmMemoryManager() self.setup() if translate_support_code: self._setup_on_leave_jitted_translated() @@ -177,6 +179,15 @@ self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) return exc + def free_loop_and_bridges(self, compiled_loop_token): + AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) + blocks = compiled_loop_token.asmmemmgr_blocks + if blocks is not None: + compiled_loop_token.asmmemmgr_blocks = None + for rawstart, rawstop in blocks: + self.gc_ll_descr.freeing_block(rawstart, rawstop) + self.asmmemmgr.free(rawstart, rawstop) + # ------------------- helpers and descriptions -------------------- @staticmethod Modified: pypy/branch/jitypes2/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/llsupport/test/test_gc.py Fri Dec 3 11:04:29 2010 @@ -9,6 +9,7 @@ from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.jit.metainterp.test.test_optimizeopt import equaloplists +from pypy.rpython.memory.gctransform import asmgcroot def test_boehm(): gc_ll_descr = GcLLDescr_boehm(None, None, None) @@ -62,58 +63,169 @@ for i in range(len(allocs)): assert addrs[i].address[0] == llmemory.cast_ptr_to_adr(allocs[i]) -def test_GcRootMap_asmgcc(): - def frame_pos(n): - return -4*(4+n) - gcrootmap = GcRootMap_asmgcc() - num1 = frame_pos(-5) - num1a = num1|2 - num2 = frame_pos(55) - num2a = ((-num2|3) >> 7) | 128 - num2b = (-num2|3) & 127 - shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, num1) - gcrootmap.add_ebp_offset(shape, num2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) - gcrootmap.add_callee_save_reg(shape, 1) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4]) - gcrootmap.add_callee_save_reg(shape, 2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8]) - gcrootmap.add_callee_save_reg(shape, 3) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12]) - gcrootmap.add_callee_save_reg(shape, 4) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12, 16]) - # - shapeaddr = gcrootmap.compress_callshape(shape) - PCALLSHAPE = lltype.Ptr(GcRootMap_asmgcc.CALLSHAPE_ARRAY) - p = llmemory.cast_adr_to_ptr(shapeaddr, PCALLSHAPE) - for i, expected in enumerate([16, 12, 8, 4, - num2a, num2b, num1a, 0, 2, 15, 11, 7, 6]): - assert p[i] == expected - # - retaddr = rffi.cast(llmemory.Address, 1234567890) - gcrootmap.put(retaddr, shapeaddr) - assert gcrootmap._gcmap[0] == retaddr - assert gcrootmap._gcmap[1] == shapeaddr - assert gcrootmap.gcmapstart().address[0] == retaddr - # - # the same as before, but enough times to trigger a few resizes - expected_shapeaddr = {} - for i in range(1, 700): +class TestGcRootMapAsmGcc: + + def test_make_shapes(self): + def frame_pos(n): + return -4*(4+n) + gcrootmap = GcRootMap_asmgcc() + num1 = frame_pos(-5) + num1a = num1|2 + num2 = frame_pos(55) + num2a = ((-num2|3) >> 7) | 128 + num2b = (-num2|3) & 127 shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, frame_pos(i)) - shapeaddr = gcrootmap.compress_callshape(shape) - expected_shapeaddr[i] = shapeaddr - retaddr = rffi.cast(llmemory.Address, 123456789 + i) + gcrootmap.add_ebp_offset(shape, num1) + gcrootmap.add_ebp_offset(shape, num2) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) + gcrootmap.add_callee_save_reg(shape, 1) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4]) + gcrootmap.add_callee_save_reg(shape, 2) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8]) + gcrootmap.add_callee_save_reg(shape, 3) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8, 12]) + gcrootmap.add_callee_save_reg(shape, 4) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8, 12, 16]) + + def test_compress_callshape(self): + class FakeDataBlockWrapper: + def malloc_aligned(self, size, alignment): + assert alignment == 1 # here + assert size == 4 + return rffi.cast(lltype.Signed, p) + datablockwrapper = FakeDataBlockWrapper() + p = lltype.malloc(rffi.CArray(lltype.Char), 4, immortal=True) + gcrootmap = GcRootMap_asmgcc() + shape = ['a', 'b', 'c', 'd'] + gcrootmap.compress_callshape(shape, datablockwrapper) + assert p[0] == 'd' + assert p[1] == 'c' + assert p[2] == 'b' + assert p[3] == 'a' + + def test_put_basic(self): + gcrootmap = GcRootMap_asmgcc() + retaddr = 1234567890 + shapeaddr = 51627384 gcrootmap.put(retaddr, shapeaddr) - for i in range(1, 700): - expected_retaddr = rffi.cast(llmemory.Address, 123456789 + i) - assert gcrootmap._gcmap[i*2+0] == expected_retaddr - assert gcrootmap._gcmap[i*2+1] == expected_shapeaddr[i] + assert gcrootmap._gcmap[0] == retaddr + assert gcrootmap._gcmap[1] == shapeaddr + p = rffi.cast(rffi.LONGP, gcrootmap.gcmapstart()) + assert p[0] == retaddr + assert (gcrootmap.gcmapend() == + gcrootmap.gcmapstart() + rffi.sizeof(lltype.Signed) * 2) + + def test_put_resize(self): + # the same as before, but enough times to trigger a few resizes + gcrootmap = GcRootMap_asmgcc() + for i in range(700): + shapeaddr = i * 100 + 1 + retaddr = 123456789 + i + gcrootmap.put(retaddr, shapeaddr) + for i in range(700): + assert gcrootmap._gcmap[i*2+0] == 123456789 + i + assert gcrootmap._gcmap[i*2+1] == i * 100 + 1 + + def test_remove_nulls(self): + expected = [] + def check(): + assert gcrootmap._gcmap_curlength == len(expected) * 2 + for i, (a, b) in enumerate(expected): + assert gcrootmap._gcmap[i*2] == a + assert gcrootmap._gcmap[i*2+1] == b + # + gcrootmap = GcRootMap_asmgcc() + for i in range(700): + shapeaddr = i * 100 # 0 if i == 0 + retaddr = 123456789 + i + gcrootmap.put(retaddr, shapeaddr) + if shapeaddr != 0: + expected.append((retaddr, shapeaddr)) + # at the first resize, the 0 should be removed + check() + for repeat in range(10): + # now clear up half the entries + assert len(expected) == 699 + for i in range(0, len(expected), 2): + gcrootmap._gcmap[i*2+1] = 0 + gcrootmap._gcmap_deadentries += 1 + expected = expected[1::2] + assert gcrootmap._gcmap_deadentries*6 > gcrootmap._gcmap_maxlength + # check that we can again insert 350 entries without a resize + oldgcmap = gcrootmap._gcmap + for i in range(0, 699, 2): + gcrootmap.put(515151 + i + repeat, 626262 + i) + expected.append((515151 + i + repeat, 626262 + i)) + assert gcrootmap._gcmap == oldgcmap + check() + + def test_freeing_block(self): + from pypy.jit.backend.llsupport import gc + class Asmgcroot: + arrayitemsize = 2 * llmemory.sizeof(llmemory.Address) + sort_count = 0 + def sort_gcmap(self, gcmapstart, gcmapend): + self.sort_count += 1 + def binary_search(self, gcmapstart, gcmapend, startaddr): + i = 0 + while (i < gcrootmap._gcmap_curlength//2 and + gcrootmap._gcmap[i*2] < startaddr): + i += 1 + if i > 0: + i -= 1 + assert 0 <= i < gcrootmap._gcmap_curlength//2 + p = rffi.cast(rffi.CArrayPtr(llmemory.Address), gcmapstart) + p = rffi.ptradd(p, 2*i) + return llmemory.cast_ptr_to_adr(p) + saved = gc.asmgcroot + try: + gc.asmgcroot = Asmgcroot() + # + gcrootmap = GcRootMap_asmgcc() + gcrootmap._gcmap = lltype.malloc(gcrootmap.GCMAP_ARRAY, + 1400, flavor='raw', + immortal=True) + for i in range(700): + gcrootmap._gcmap[i*2] = 1200000 + i + gcrootmap._gcmap[i*2+1] = i * 100 + 1 + assert gcrootmap._gcmap_deadentries == 0 + assert gc.asmgcroot.sort_count == 0 + gcrootmap._gcmap_maxlength = 1400 + gcrootmap._gcmap_curlength = 1400 + gcrootmap._gcmap_sorted = False + # + gcrootmap.freeing_block(1200000 - 100, 1200000) + assert gcrootmap._gcmap_deadentries == 0 + assert gc.asmgcroot.sort_count == 1 + # + gcrootmap.freeing_block(1200000 + 100, 1200000 + 200) + assert gcrootmap._gcmap_deadentries == 100 + assert gc.asmgcroot.sort_count == 1 + for i in range(700): + if 100 <= i < 200: + expected = 0 + else: + expected = i * 100 + 1 + assert gcrootmap._gcmap[i*2] == 1200000 + i + assert gcrootmap._gcmap[i*2+1] == expected + # + gcrootmap.freeing_block(1200000 + 650, 1200000 + 750) + assert gcrootmap._gcmap_deadentries == 150 + assert gc.asmgcroot.sort_count == 1 + for i in range(700): + if 100 <= i < 200 or 650 <= i: + expected = 0 + else: + expected = i * 100 + 1 + assert gcrootmap._gcmap[i*2] == 1200000 + i + assert gcrootmap._gcmap[i*2+1] == expected + # + finally: + gc.asmgcroot = saved class FakeLLOp(object): Modified: pypy/branch/jitypes2/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/model.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/model.py Fri Dec 3 11:04:29 2010 @@ -1,3 +1,4 @@ +from pypy.rlib.debug import debug_start, debug_print, debug_stop from pypy.jit.metainterp import history, compile @@ -7,17 +8,27 @@ done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + total_compiled_loops = 0 + total_compiled_bridges = 0 + total_freed_loops = 0 + total_freed_bridges = 0 def __init__(self): self.fail_descr_list = [] + self.fail_descr_free_list = [] def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) n = descr.index if n < 0: lst = self.fail_descr_list - n = len(lst) - lst.append(descr) + if len(self.fail_descr_free_list) > 0: + n = self.fail_descr_free_list.pop() + assert lst[n] is None + lst[n] = descr + else: + n = len(lst) + lst.append(descr) descr.index = n return n @@ -35,12 +46,14 @@ def compile_loop(self, inputargs, operations, looptoken, log=True): """Assemble the given loop. - Extra attributes should be put in the LoopToken to - point to the compiled loop in assembler. + Should create and attach a fresh CompiledLoopToken to + looptoken.compiled_loop_token and stick extra attributes + on it to point to the compiled loop in assembler. """ raise NotImplementedError - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): """Assemble the bridge. The FailDescr is the descr of the original guard that failed. """ @@ -113,6 +126,28 @@ oldlooptoken so that from now own they will call newlooptoken.""" raise NotImplementedError + def free_loop_and_bridges(self, compiled_loop_token): + """This method is called to free resources (machine code, + references to resume guards, etc.) allocated by the compilation + of a loop and all bridges attached to it. After this call, the + frontend cannot use this compiled loop any more; in fact, it + guarantees that at the point of the call to free_code_group(), + none of the corresponding assembler is currently running. + """ + # The base class provides a limited implementation: freeing the + # resume descrs. This is already quite helpful, because the + # resume descrs are the largest consumers of memory (about 3x + # more than the assembler, in the case of the x86 backend). + lst = self.fail_descr_list + # We expect 'compiled_loop_token' to be itself garbage-collected soon, + # but better safe than sorry: be ready to handle several calls to + # free_loop_and_bridges() for the same compiled_loop_token. + faildescr_indices = compiled_loop_token.faildescr_indices + compiled_loop_token.faildescr_indices = [] + for n in faildescr_indices: + lst[n] = None + self.fail_descr_free_list.extend(faildescr_indices) + @staticmethod def sizeof(S): raise NotImplementedError @@ -237,3 +272,40 @@ def force(self, force_token): raise NotImplementedError + + +class CompiledLoopToken(object): + asmmemmgr_blocks = None + asmmemmgr_gcroots = 0 + + def __init__(self, cpu, number): + cpu.total_compiled_loops += 1 + self.cpu = cpu + self.number = number + self.bridges_count = 0 + # This growing list gives the 'descr_number' of all fail descrs + # that belong to this loop or to a bridge attached to it. + # Filled by the frontend calling record_faildescr_index(). + self.faildescr_indices = [] + debug_start("jit-mem-looptoken-alloc") + debug_print("allocating Loop #", self.number) + debug_stop("jit-mem-looptoken-alloc") + + def record_faildescr_index(self, n): + self.faildescr_indices.append(n) + + def compiling_a_bridge(self): + self.cpu.total_compiled_bridges += 1 + self.bridges_count += 1 + debug_start("jit-mem-looptoken-alloc") + debug_print("allocating Bridge #", self.bridges_count, "of Loop #", self.number) + debug_stop("jit-mem-looptoken-alloc") + + def __del__(self): + debug_start("jit-mem-looptoken-free") + debug_print("freeing Loop #", self.number, 'with', + self.bridges_count, 'attached bridges') + self.cpu.free_loop_and_bridges(self) + self.cpu.total_freed_loops += 1 + self.cpu.total_freed_bridges += self.bridges_count + debug_stop("jit-mem-looptoken-free") Modified: pypy/branch/jitypes2/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/test/runner_test.py Fri Dec 3 11:04:29 2010 @@ -174,6 +174,8 @@ assert not wr_i1() and not wr_guard() def test_compile_bridge(self): + self.cpu.total_compiled_loops = 0 + self.cpu.total_compiled_bridges = 0 i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() @@ -199,7 +201,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) @@ -207,6 +209,10 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + assert self.cpu.total_compiled_loops == 1 + assert self.cpu.total_compiled_bridges == 1 + return looptoken + def test_compile_bridge_with_holes(self): i0 = BoxInt() i1 = BoxInt() @@ -233,7 +239,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) @@ -1050,7 +1056,7 @@ ResOperation(rop.JUMP, [f3] + fboxes2[1:], None, descr=looptoken), ] - self.cpu.compile_bridge(faildescr1, fboxes2, bridge) + self.cpu.compile_bridge(faildescr1, fboxes2, bridge, looptoken) for i in range(len(fboxes)): self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) @@ -1197,6 +1203,13 @@ yield nan_and_infinity, rop.FLOAT_GT, operator.gt, all_cases_binary yield nan_and_infinity, rop.FLOAT_GE, operator.ge, all_cases_binary + def test_noops(self): + c_box = self.alloc_string("hi there").constbox() + c_nest = ConstInt(0) + self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest], 'void') + self.execute_operation(rop.JIT_DEBUG, [c_box, c_nest, c_nest, + c_nest, c_nest], 'void') + class LLtypeBackendTest(BaseBackendTest): @@ -2224,6 +2237,20 @@ assert res.value == expected, ( "%r: got %r, expected %r" % (RESTYPE, res.value, expected)) + def test_free_loop_and_bridges(self): + from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("not a subclass of llmodel.AbstractLLCPU") + if hasattr(self.cpu, 'setup_once'): + self.cpu.setup_once() + mem0 = self.cpu.asmmemmgr.total_mallocs + looptoken = self.test_compile_bridge() + mem1 = self.cpu.asmmemmgr.total_mallocs + self.cpu.free_loop_and_bridges(looptoken.compiled_loop_token) + mem2 = self.cpu.asmmemmgr.total_mallocs + assert mem2 < mem1 + assert mem2 == mem0 + class OOtypeBackendTest(BaseBackendTest): Modified: pypy/branch/jitypes2/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/test/test_random.py Fri Dec 3 11:04:29 2010 @@ -524,7 +524,7 @@ self.prebuilt_ptr_consts = [] self.r = r self.build_random_loop(cpu, builder_factory, r, startvars) - + def build_random_loop(self, cpu, builder_factory, r, startvars): loop = TreeLoop('test_random_function') @@ -685,11 +685,12 @@ subloop.operations[-1] = jump_op self.guard_op = rl.guard_op self.prebuilt_ptr_consts += rl.prebuilt_ptr_consts + self.loop.token.record_jump_to(rl.loop.token) self.dont_generate_more = True if r.random() < .05: return False self.builder.cpu.compile_bridge(fail_descr, fail_args, - subloop.operations) + subloop.operations, self.loop.token) return True def check_random_function(cpu, BuilderClass, r, num=None, max=None): Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/assembler.py Fri Dec 3 11:04:29 2010 @@ -1,12 +1,13 @@ import sys, os from pypy.jit.backend.llsupport import symbolic +from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT, LoopToken) from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper -from pypy.tool.uid import fixid +from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, _get_scale) @@ -30,10 +31,11 @@ from pypy.jit.backend.x86 import rx86, regloc, codebuf from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array -from pypy.rlib.debug import debug_print +from pypy.jit.backend.x86 import support +from pypy.rlib.debug import (debug_print, debug_start, debug_stop, + have_debug_prints) from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout -from pypy.rlib.streamio import open_file_as_stream from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, @@ -43,123 +45,17 @@ def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) -class MachineCodeBlockWrapper(object): - MC_DEFAULT_SIZE = 1024*1024 - - def __init__(self, assembler, bigsize, profile_agent=None): - self.assembler = assembler - self.old_mcs = [] # keepalive - self.bigsize = bigsize - self._mc = self._instantiate_mc() - self.function_name = None - self.profile_agent = profile_agent - self.reset_reserved_bytes() - - def _instantiate_mc(self): # hook for testing - return codebuf.MachineCodeBlock(self.bigsize) - - def ensure_bytes_available(self, num_bytes): - if self.bytes_free() <= (self._reserved_bytes + num_bytes): - self.make_new_mc() - - def reserve_bytes(self, num_bytes): - self.ensure_bytes_available(num_bytes) - self._reserved_bytes += num_bytes - - def reset_reserved_bytes(self): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - self._reserved_bytes = 64 - - def get_relative_pos(self): - return self._mc.get_relative_pos() - - def overwrite(self, pos, listofchars): - return self._mc.overwrite(pos, listofchars) - - def bytes_free(self): - return self._mc._size - self._mc.get_relative_pos() - - def start_function(self, name): - self.function_name = name - self.start_pos = self._mc.get_relative_pos() - - def end_function(self, done=True): - assert self.function_name is not None - size = self._mc.get_relative_pos() - self.start_pos - address = self.tell() - size - if self.profile_agent is not None: - self.profile_agent.native_code_written(self.function_name, - address, size) - if done: - self.function_name = None - - def make_new_mc(self): - new_mc = self._instantiate_mc() - debug_print('[new machine code block at', new_mc.tell(), ']') - - if IS_X86_64: - # The scratch register is sometimes used as a temporary - # register, but the JMP below might clobber it. Rather than risk - # subtle bugs, we preserve the scratch register across the jump. - self._mc.PUSH_r(X86_64_SCRATCH_REG.value) - - self._mc.JMP(imm(new_mc.tell())) - - if IS_X86_64: - # Restore scratch reg - new_mc.POP_r(X86_64_SCRATCH_REG.value) - - if self.function_name is not None: - self.end_function(done=False) - self.start_pos = new_mc.get_relative_pos() - - self.assembler.write_pending_failure_recoveries() - - self._mc.done() - self.old_mcs.append(self._mc) - self._mc = new_mc - make_new_mc._dont_inline_ = True - - def tell(self): - return self._mc.tell() - - def done(self): - self._mc.done() - -def _new_method(name): - def method(self, *args): - if self.bytes_free() < self._reserved_bytes: - self.make_new_mc() - getattr(self._mc, name)(*args) - method.func_name = name - return method - -for _name in rx86.all_instructions + regloc.all_extra_instructions: - setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) - -for name in dir(codebuf.MachineCodeBlock): - if name.upper() == name or name == "writechr": - setattr(MachineCodeBlockWrapper, name, _new_method(name)) class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + def __init__(self, faildescr, failargs, fail_locs, exc): self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs self.exc = exc - self.desc_bytes = desc_bytes - - def recovery_stub_size(self): - # XXX: 32 is pulled out of the air - return 32 + len(self.desc_bytes) DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) class Assembler386(object): - mc = None - mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE - _float_constants = None _regalloc = None _output_loop_log = None @@ -177,16 +73,18 @@ self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 self.loop_run_counters = [] - # if we have 10000 loops, we have some other problems I guess self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 - self.pending_guard_tokens = None self.memcpy_addr = 0 self.setup_failure_recovery() self._debug = False self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.fail_boxes_count = 0 + self._current_depths_cache = (0, 0) + self.datablockwrapper = None + self.teardown() def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -196,67 +94,65 @@ def set_debug(self, v): self._debug = v - def setup(self): - if self.mc is None: - # the address of the function called by 'new' - gc_ll_descr = self.cpu.gc_ll_descr - gc_ll_descr.initialize() - ll_new = gc_ll_descr.get_funcptr_for_new() - self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) - if gc_ll_descr.get_funcptr_for_newarray is not None: - ll_new_array = gc_ll_descr.get_funcptr_for_newarray() - self.malloc_array_func_addr = rffi.cast(lltype.Signed, - ll_new_array) - if gc_ll_descr.get_funcptr_for_newstr is not None: - ll_new_str = gc_ll_descr.get_funcptr_for_newstr() - self.malloc_str_func_addr = rffi.cast(lltype.Signed, - ll_new_str) - if gc_ll_descr.get_funcptr_for_newunicode is not None: - ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() - self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, - ll_new_unicode) - self.memcpy_addr = self.cpu.cast_ptr_to_int(codebuf.memcpy_fn) - self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) - self._build_failure_recovery(False) - self._build_failure_recovery(True) - if self.cpu.supports_floats: - self._build_failure_recovery(False, withfloats=True) - self._build_failure_recovery(True, withfloats=True) - codebuf.ensure_sse2_floats() - self._build_float_constants() - if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): - self._build_malloc_fixedsize_slowpath() - s = os.environ.get('PYPYLOG') - if s: - if s.find(':') != -1: - s = s.split(':')[-1] - self.set_debug(True) - self._output_loop_log = s + ".count" - # Intialize here instead of __init__ to prevent - # pending_guard_tokens from being considered a prebuilt object, - # which sometimes causes memory leaks since the prebuilt list is - # still considered a GC root after we re-assign - # pending_guard_tokens in write_pending_failure_recoveries - self.pending_guard_tokens = [] + def setup_once(self): + # the address of the function called by 'new' + gc_ll_descr = self.cpu.gc_ll_descr + gc_ll_descr.initialize() + ll_new = gc_ll_descr.get_funcptr_for_new() + self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) + if gc_ll_descr.get_funcptr_for_newarray is not None: + ll_new_array = gc_ll_descr.get_funcptr_for_newarray() + self.malloc_array_func_addr = rffi.cast(lltype.Signed, + ll_new_array) + if gc_ll_descr.get_funcptr_for_newstr is not None: + ll_new_str = gc_ll_descr.get_funcptr_for_newstr() + self.malloc_str_func_addr = rffi.cast(lltype.Signed, + ll_new_str) + if gc_ll_descr.get_funcptr_for_newunicode is not None: + ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() + self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, + ll_new_unicode) + self.memcpy_addr = self.cpu.cast_ptr_to_int(support.memcpy_fn) + self._build_failure_recovery(False) + self._build_failure_recovery(True) + if self.cpu.supports_floats: + self._build_failure_recovery(False, withfloats=True) + self._build_failure_recovery(True, withfloats=True) + support.ensure_sse2_floats() + self._build_float_constants() + if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): + self._build_malloc_fixedsize_slowpath() + debug_start('jit-backend-counts') + self.set_debug(have_debug_prints()) + debug_stop('jit-backend-counts') + + def setup(self, looptoken): + assert self.memcpy_addr != 0, "setup_once() not called?" + self.pending_guard_tokens = [] + self.mc = codebuf.MachineCodeBlockWrapper() + if self.datablockwrapper is None: + allblocks = self.get_asmmemmgr_blocks(looptoken) + self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, + allblocks) + + def teardown(self): + self.pending_guard_tokens = None + self.mc = None + self.looppos = -1 + self.currently_compiling_loop = None def finish_once(self): if self._debug: - output_log = self._output_loop_log - assert output_log is not None - f = open_file_as_stream(output_log, "w") + debug_start('jit-backend-counts') for i in range(len(self.loop_run_counters)): - name, struct = self.loop_run_counters[i] - f.write(str(name) + ":" + str(struct.i) + "\n") - f.close() + struct = self.loop_run_counters[i] + debug_print(str(i) + ':' + str(struct.i)) + debug_stop('jit-backend-counts') def _build_float_constants(self): - # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment - addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw', - track_allocation=False) - if not we_are_translated(): - self._keepalive_malloced_float_consts = addr - float_constants = rffi.cast(lltype.Signed, addr) - float_constants = (float_constants + 15) & ~15 # align to 16 bytes + datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, []) + float_constants = datablockwrapper.malloc_aligned(32, alignment=16) + datablockwrapper.done() addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' # 0x8000000000000000 @@ -271,45 +167,56 @@ def _build_malloc_fixedsize_slowpath(self): # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = self.mc.tell() + mc = codebuf.MachineCodeBlockWrapper() if self.cpu.supports_floats: # save the XMM registers in for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 - self.mc.MOVSD_sx((WORD*2)+8*i, i) - self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + mc.MOVSD_sx((WORD*2)+8*i, i) + mc.SUB_rr(edx.value, eax.value) # compute the size we want if IS_X86_32: - self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + mc.MOV_sr(WORD, edx.value) # save it as the new argument elif IS_X86_64: # rdi can be clobbered: its content was forced to the stack # by _fastpath_malloc(), like all other save_around_call_regs. - self.mc.MOV_rr(edi.value, edx.value) + mc.MOV_rr(edi.value, edx.value) addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - self.mc.JMP(imm(addr)) # tail call to the real malloc + mc.JMP(imm(addr)) # tail call to the real malloc + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.malloc_fixedsize_slowpath1 = rawstart # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = self.mc.tell() + mc = codebuf.MachineCodeBlockWrapper() if self.cpu.supports_floats: # restore the XMM registers for i in range(self.cpu.NUM_REGS):# from where they were saved - self.mc.MOVSD_xs(i, (WORD*2)+8*i) + mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - self.mc.RET() - self.mc.done() + mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + mc.RET() + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.malloc_fixedsize_slowpath2 = rawstart def assemble_loop(self, inputargs, operations, looptoken, log): - """adds the following attributes to looptoken: + '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) _x86_bootstrap_code (an integer giving an address) - _x86_direct_bootstrap_code + _x86_direct_bootstrap_code ( " " " " ) _x86_frame_depth _x86_param_depth _x86_arglocs _x86_debug_checksum - """ + ''' + # XXX this function is too longish and contains some code + # duplication with assemble_bridge(). Also, we should think + # about not storing on 'self' attributes that will live only + # for the duration of compiling one loop or a one bridge. + + clt = CompiledLoopToken(self.cpu, looptoken.number) + looptoken.compiled_loop_token = clt if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.setup() + self.setup(looptoken) + self.currently_compiling_loop = looptoken funcname = self._find_debug_merge_point(operations) if log: self._register_counter() @@ -319,43 +226,61 @@ arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - # profile support - name = "Loop # %s: %s" % (looptoken.number, funcname) - self.mc.start_function(name) - looptoken._x86_bootstrap_code = self.mc.tell() - adr_stackadjust = self._assemble_bootstrap_code(inputargs, arglocs) - curadr = self.mc.tell() - looptoken._x86_loop_code = curadr + bootstrappos = self.mc.get_relative_pos() + stackadjustpos = self._assemble_bootstrap_code(inputargs, arglocs) + self.looppos = self.mc.get_relative_pos() looptoken._x86_frame_depth = -1 # temporarily looptoken._x86_param_depth = -1 # temporarily frame_depth, param_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - looptoken._x86_direct_bootstrap_code = self.mc.tell() - self._assemble_bootstrap_direct_call(arglocs, curadr, + directbootstrappos = self.mc.get_relative_pos() + self._assemble_bootstrap_direct_call(arglocs, self.looppos, frame_depth+param_depth) - # - debug_print("Loop #%d has address %x to %x" % (looptoken.number, - looptoken._x86_loop_code, - self.mc.tell())) - self.mc.end_function() self.write_pending_failure_recoveries() - - def assemble_bridge(self, faildescr, inputargs, operations, log): + fullsize = self.mc.get_relative_pos() + # + rawstart = self.materialize_loop(looptoken) + debug_print("Loop #%d (%s) has address %x to %x" % ( + looptoken.number, funcname, + rawstart + self.looppos, + rawstart + directbootstrappos)) + self._patch_stackadjust(rawstart + stackadjustpos, + frame_depth + param_depth) + self.patch_pending_failure_recoveries(rawstart) + # + looptoken._x86_bootstrap_code = rawstart + bootstrappos + looptoken._x86_loop_code = rawstart + self.looppos + looptoken._x86_direct_bootstrap_code = rawstart + directbootstrappos + self.teardown() + # oprofile support + if self.cpu.profile_agent is not None: + name = "Loop # %s: %s" % (looptoken.number, funcname) + self.cpu.profile_agent.native_code_written(name, + rawstart, fullsize) + + def assemble_bridge(self, faildescr, inputargs, operations, + original_loop_token, log): if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.setup() + descr_number = self.cpu.get_fail_descr_number(faildescr) + try: + failure_recovery = self._find_failure_recovery_bytecode(faildescr) + except ValueError: + debug_print("Bridge out of guard", descr_number, + "was already compiled!") + return + + self.setup(original_loop_token) funcname = self._find_debug_merge_point(operations) if log: self._register_counter() operations = self._inject_debugging_code(faildescr, operations) - arglocs = self.rebuild_faillocs_from_descr( - faildescr._x86_failure_recovery_bytecode) + arglocs = self.rebuild_faillocs_from_descr(failure_recovery) if not we_are_translated(): assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) @@ -364,37 +289,63 @@ regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) - # oprofile support - descr_number = self.cpu.get_fail_descr_number(faildescr) - name = "Bridge # %s: %s" % (descr_number, funcname) - self.mc.start_function(name) - - adr_bridge = self.mc.tell() - adr_stackadjust = self._patchable_stackadjust() + stackadjustpos = self._patchable_stackadjust() frame_depth, param_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) + codeendpos = self.mc.get_relative_pos() + self.write_pending_failure_recoveries() + fullsize = self.mc.get_relative_pos() + # + rawstart = self.materialize_loop(original_loop_token) + + debug_print("Bridge out of guard %d (%s) has address %x to %x" % + (descr_number, funcname, rawstart, rawstart + codeendpos)) + self._patch_stackadjust(rawstart + stackadjustpos, + frame_depth + param_depth) + self.patch_pending_failure_recoveries(rawstart) if not we_are_translated(): # for the benefit of tests faildescr._x86_bridge_frame_depth = frame_depth faildescr._x86_bridge_param_depth = param_depth # patch the jump from original guard - self.patch_jump_for_descr(faildescr, adr_bridge) - debug_print("Bridge out of guard %d has address %x to %x" % - (descr_number, adr_bridge, self.mc.tell())) - self.mc.end_function() - self.write_pending_failure_recoveries() + self.patch_jump_for_descr(faildescr, rawstart) + self.teardown() + # oprofile support + if self.cpu.profile_agent is not None: + name = "Bridge # %s: %s" % (descr_number, funcname) + self.cpu.profile_agent.native_code_written(name, + rawstart, fullsize) def write_pending_failure_recoveries(self): + # for each pending guard, generate the code of the recovery stub + # at the end of self.mc. for tok in self.pending_guard_tokens: - # Okay to write to _mc because we've already made sure that - # there's enough space by "reserving" bytes. - addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) - tok.faildescr._x86_adr_recovery_stub = addr - self.patch_jump_for_descr(tok.faildescr, addr) + tok.pos_recovery_stub = self.generate_quick_failure(tok) - self.pending_guard_tokens = [] - self.mc.reset_reserved_bytes() - self.mc.done() + def patch_pending_failure_recoveries(self, rawstart): + # after we wrote the assembler to raw memory, set up + # tok.faildescr._x86_adr_jump_offset to contain the raw address of + # the 4-byte target field in the JMP/Jcond instruction, and patch + # the field in question to point (initially) to the recovery stub + for tok in self.pending_guard_tokens: + addr = rawstart + tok.pos_jump_offset + tok.faildescr._x86_adr_jump_offset = addr + relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4) + assert rx86.fits_in_32bits(relative_target) + p = rffi.cast(rffi.INTP, addr) + p[0] = rffi.cast(rffi.INT, relative_target) + + def get_asmmemmgr_blocks(self, looptoken): + clt = looptoken.compiled_loop_token + if clt.asmmemmgr_blocks is None: + clt.asmmemmgr_blocks = [] + return clt.asmmemmgr_blocks + + def materialize_loop(self, looptoken): + self.datablockwrapper.done() # finish using cpu.asmmemmgr + self.datablockwrapper = None + allblocks = self.get_asmmemmgr_blocks(looptoken) + return self.mc.materialize(self.cpu.asmmemmgr, allblocks, + self.cpu.gc_ll_descr.gcrootmap) def _find_debug_merge_point(self, operations): @@ -409,29 +360,50 @@ def _register_counter(self): if self._debug: + # YYY very minor leak -- we need the counters to stay alive + # forever, just because we want to report them at the end + # of the process struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', - track_allocation=False) # known to leak + track_allocation=False) struct.i = 0 - self.loop_run_counters.append((len(self.loop_run_counters), struct)) - + self.loop_run_counters.append(struct) + + def _find_failure_recovery_bytecode(self, faildescr): + adr_jump_offset = faildescr._x86_adr_jump_offset + if adr_jump_offset == 0: + raise ValueError + # follow the JMP/Jcond + p = rffi.cast(rffi.INTP, adr_jump_offset) + adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) + # skip the CALL + if WORD == 4: + adr_target += 5 # CALL imm + else: + adr_target += 13 # MOV r11, imm; CALL *r11 + return adr_target + def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - adr_recovery_stub = faildescr._x86_adr_recovery_stub + assert adr_jump_offset != 0 offset = adr_new_target - (adr_jump_offset + 4) # If the new target fits within a rel32 of the jump, just patch # that. Otherwise, leave the original rel32 to the recovery stub in # place, but clobber the recovery stub with a jump to the real # target. + mc = codebuf.MachineCodeBlockWrapper() if rx86.fits_in_32bits(offset): - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) mc.writeimm32(offset) + mc.copy_to_raw_memory(adr_jump_offset) else: - # "mov r11, addr; jmp r11" is 13 bytes - mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + # "mov r11, addr; jmp r11" is 13 bytes, which fits in there + # because we always write "mov r11, addr; call *r11" in the + # first place. mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) mc.JMP_r(X86_64_SCRATCH_REG.value) - - mc.done() + p = rffi.cast(rffi.INTP, adr_jump_offset) + adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) + mc.copy_to_raw_memory(adr_target) + faildescr._x86_adr_jump_offset = 0 # means "patched" @specialize.argtype(1) def _inject_debugging_code(self, looptoken, operations): @@ -442,7 +414,7 @@ s += op.getopnum() looptoken._x86_debug_checksum = s c_adr = ConstInt(rffi.cast(lltype.Signed, - self.loop_run_counters[-1][1])) + self.loop_run_counters[-1])) box = BoxInt() box2 = BoxInt() ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], @@ -451,19 +423,11 @@ ResOperation(rop.SETFIELD_RAW, [c_adr, box2], None, descr=self.debug_counter_descr)] operations = ops + operations - # # we need one register free (a bit of a hack, but whatever) - # self.mc.PUSH(eax) - # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) - # self.mc.MOV(eax, heap(adr)) - # self.mc.ADD(eax, imm1) - # self.mc.MOV(heap(adr), eax) - # self.mc.POP(eax) return operations def _assemble(self, regalloc, operations): self._regalloc = regalloc regalloc.walk_operations(operations) - self.mc.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -479,30 +443,31 @@ def _patchable_stackadjust(self): # stack adjustment LEA self.mc.LEA32_rb(esp.value, 0) - return self.mc.tell() - 4 + return self.mc.get_relative_pos() - 4 - def _patch_stackadjust(self, adr_lea, reserved_depth): + def _patch_stackadjust(self, adr_lea, allocated_depth): # patch stack adjustment LEA - mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) - # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. + mc = codebuf.MachineCodeBlockWrapper() + # Compute the correct offset for the instruction LEA ESP, [EBP-4*words] + mc.writeimm32(self._get_offset_of_ebp_from_esp(allocated_depth)) + mc.copy_to_raw_memory(adr_lea) + + def _get_offset_of_ebp_from_esp(self, allocated_depth): # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth - # align, e.g. for Mac OS X + words = (self.cpu.FRAME_FIXED_SIZE - 1) + allocated_depth + # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.writeimm32(-WORD * aligned_words) - mc.done() + return -WORD * aligned_words def _call_header(self): + # NB. the shape of the frame is hard-coded in get_basic_shape() too. + # Also, make sure this is consistent with FRAME_FIXED_SIZE. self.mc.PUSH_r(ebp.value) self.mc.MOV_rr(ebp.value, esp.value) for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) - # NB. the shape of the frame is hard-coded in get_basic_shape() too. - # Also, make sure this is consistent with FRAME_FIXED_SIZE. - return self._patchable_stackadjust() - def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -512,17 +477,16 @@ self.mc.POP_r(ebp.value) self.mc.RET() - def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + def _assemble_bootstrap_direct_call(self, arglocs, jmppos, stackdepth): if IS_X86_64: - return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) + return self._assemble_bootstrap_direct_call_64(arglocs, jmppos, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - # XXX not to repeat the logic, a bit around - adr_stackadjust = self._call_header() - self._patch_stackadjust(adr_stackadjust, stackdepth) + self._call_header() + self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if isinstance(loc, RegLoc): @@ -544,9 +508,11 @@ self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) assert isinstance(loc, StackLoc) self.mc.MOVSD_bx(loc.value, xmmtmp.value) - self.mc.JMP_l(jmpadr) + endpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(jmppos - endpos) + assert endpos == self.mc.get_relative_pos() - def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + def _assemble_bootstrap_direct_call_64(self, arglocs, jmppos, stackdepth): # XXX: Very similar to _emit_call_64 src_locs = [] @@ -560,8 +526,8 @@ unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - adr_stackadjust = self._call_header() - self._patch_stackadjust(adr_stackadjust, stackdepth) + self._call_header() + self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones assert len(nonfloatlocs) == len(floatlocs) @@ -597,8 +563,9 @@ # clobber the scratch register self.mc.MOV(loc, X86_64_SCRATCH_REG) - finaljmp = self.mc.tell() - self.mc.JMP(imm(jmpadr)) + endpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(jmppos - endpos) + assert endpos == self.mc.get_relative_pos() def redirect_call_assembler(self, oldlooptoken, newlooptoken): # some minimal sanity checking @@ -611,16 +578,17 @@ # Ideally we should rather patch all existing CALLs, but well. oldadr = oldlooptoken._x86_direct_bootstrap_code target = newlooptoken._x86_direct_bootstrap_code - mc = codebuf.InMemoryCodeBuilder(oldadr, oldadr + 16) + mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) - mc.done() + mc.copy_to_raw_memory(oldadr) def _assemble_bootstrap_code(self, inputargs, arglocs): nonfloatlocs, floatlocs = arglocs - adr_stackadjust = self._call_header() + self._call_header() + stackadjustpos = self._patchable_stackadjust() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] - self.mc._mc.begin_reuse_scratch_register() + self.mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: @@ -652,8 +620,8 @@ self.mc.MOVSD(xmmtmp, heap(adr)) assert isinstance(loc, StackLoc) self.mc.MOVSD_bx(loc.value, xmmtmp.value) - self.mc._mc.end_reuse_scratch_register() - return adr_stackadjust + self.mc.end_reuse_scratch_register() + return stackadjustpos def dump(self, text): if not self.verbose: @@ -661,7 +629,8 @@ _prev = Box._extended_display try: Box._extended_display = False - print >> sys.stderr, ' 0x%x %s' % (fixid(self.mc.tell()), text) + pos = self.mc.get_relative_pos() + print >> sys.stderr, ' 0x%x %s' % (pos, text) finally: Box._extended_display = _prev @@ -721,7 +690,7 @@ arglocs, resloc) if not we_are_translated(): # must be added by the genop_guard_list[]() - assert hasattr(faildescr, '_x86_adr_jump_offset') + assert guard_token is self.pending_guard_tokens[-1] def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc, current_depths): @@ -794,9 +763,6 @@ result_loc): guard_opnum = guard_op.getopnum() self.mc.UCOMISD(arglocs[0], arglocs[1]) - # 16 is enough space for the rel8 jumps below and the rel32 - # jump in implement_guard - self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: if need_jp: self.mc.J_il8(rx86.Conditions['P'], 6) @@ -964,9 +930,6 @@ def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.getopnum() self.mc.UCOMISD(arglocs[0], arglocs[1]) - # 16 is enough space for the rel8 jumps below and the rel32 - # jump in implement_guard - self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: self.mc.J_il8(rx86.Conditions['P'], 6) self.implement_guard(guard_token, 'E') @@ -1290,13 +1253,11 @@ self.mc.CMP32_mi((locs[0].value, 0), expected_typeid) def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): - self.mc.ensure_bytes_available(256) self._cmp_guard_class(locs) self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, guard_token, locs, ign_2): - self.mc.ensure_bytes_available(256) self.mc.CMP(locs[0], imm1) # Patched below self.mc.J_il8(rx86.Conditions['B'], 0) @@ -1305,7 +1266,7 @@ # patch the JB above offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, [chr(offset)]) + self.mc.overwrite(jb_location-1, chr(offset)) # self.implement_guard(guard_token, 'NE') @@ -1314,42 +1275,33 @@ exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - desc_bytes = self.failure_recovery_description(failargs, fail_locs) - return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) + return GuardToken(faildescr, failargs, fail_locs, exc) - def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): + def generate_quick_failure(self, guardtok): """Generate the initial code for handling a failure. We try to - keep it as compact as possible. The idea is that this code is - executed at most once (and very often, zero times); when - executed, it generates a more complete piece of code which can - really handle recovery from this particular failure. + keep it as compact as possible. """ - fail_index = self.cpu.get_fail_descr_number(faildescr) - addr = mc.tell() + fail_index = self.cpu.get_fail_descr_number(guardtok.faildescr) + mc = self.mc + startpos = mc.get_relative_pos() withfloats = False - for box in failargs: + for box in guardtok.failargs: if box is not None and box.type == FLOAT: withfloats = True break + exc = guardtok.exc mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery - faildescr._x86_failure_recovery_bytecode = mc.tell() - for byte in desc_bytes: - mc.writechr(ord(byte)) + self.write_failure_recovery_description(mc, guardtok.failargs, + guardtok.fail_locs) # write the fail_index too mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): - mc.writechr(0xCC) - faildescr._x86_debug_faillocs = [loc for loc in fail_locs - if loc is not None] - - # Make sure the recovery stub is at least 16 bytes long (for the - # case where we overwrite the recovery stub with a 64-bit absolute - # jump) - while mc.tell() - addr < 16: - mc.writechr(0x00) - return addr + mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.fail_locs if loc is not None] + guardtok.faildescr._x86_debug_faillocs = faillocs + return startpos DESCR_REF = 0x00 DESCR_INT = 0x01 @@ -1360,8 +1312,7 @@ CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def failure_recovery_description(self, failargs, locs): - desc_bytes = [] + def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1381,19 +1332,14 @@ n = loc.value n = kind + 4*n while n > 0x7F: - desc_bytes.append(chr((n & 0x7F) | 0x80)) + mc.writechar(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - desc_bytes.append(chr(n)) - desc_bytes.append(chr(self.CODE_STOP)) + mc.writechar(chr(n)) + mc.writechar(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE - return desc_bytes - - def write_failure_recovery_description(self, mc, failargs, locs): - for byte in self.failure_recovery_description(failargs, locs): - mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager @@ -1531,10 +1477,8 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc._mc - # Assume that we are called at the beginning, when there is no risk - # that 'mc' runs out of space. Checked by asserts in mc.write(). - recovery_addr = mc.tell() + mc = codebuf.MachineCodeBlockWrapper() + self.mc = mc # Push all general purpose registers for gpr in range(self.cpu.NUM_REGS-1, -1, -1): @@ -1585,11 +1529,12 @@ # above. self._call_footer() - self.mc.done() - self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None def generate_failure(self, fail_index, locs, exc, locs_are_ref): - self.mc._mc.begin_reuse_scratch_register() + self.mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] if isinstance(loc, RegLoc): @@ -1616,7 +1561,7 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(eax, loc) self.mc.MOV(heap(adr), eax) - self.mc._mc.end_reuse_scratch_register() + self.mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1632,17 +1577,13 @@ self._call_footer() def implement_guard(self, guard_token, condition=None): - self.mc.reserve_bytes(guard_token.recovery_stub_size()) - self.pending_guard_tokens.append(guard_token) - # These jumps are patched later, the mc.tell() are just - # dummy values. Also, use self.mc._mc to avoid triggering a - # "buffer full" exactly here. - mc = self.mc._mc + # These jumps are patched later. if condition: - mc.J_il(rx86.Conditions[condition], mc.tell()) + self.mc.J_il(rx86.Conditions[condition], 0) else: - mc.JMP_l(mc.tell()) - guard_token.faildescr._x86_adr_jump_offset = mc.tell() - 4 + self.mc.JMP_l(0) + guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4 + self.pending_guard_tokens.append(guard_token) def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] @@ -1697,7 +1638,6 @@ # Write a call to the direct_bootstrap_code of the target assembler self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1733,7 +1673,7 @@ # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - self.mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, chr(offset)) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1768,7 +1708,7 @@ # Here we join Path A and Path B again offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.overwrite(jmp_location - 1, chr(offset)) self.mc.CMP_bi(FORCE_INDEX_OFS, 0) self.implement_guard(guard_token, 'L') @@ -1783,9 +1723,6 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - # ensure that enough bytes are available to write the whole - # following piece of code atomically (for the JZ) - self.mc.ensure_bytes_available(256) self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), descr.jit_wb_if_flag_singlebyte) self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later @@ -1828,7 +1765,7 @@ # patch the JZ above offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - self.mc.overwrite(jz_location-1, [chr(offset)]) + self.mc.overwrite(jz_location-1, chr(offset)) def genop_force_token(self, op, arglocs, resloc): # RegAlloc.consider_force_token ensures this: @@ -1851,18 +1788,21 @@ gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: mark = self._regalloc.get_mark_gc_roots(gcrootmap) - gcrootmap.put(rffi.cast(llmemory.Address, self.mc.tell()), mark) + self.mc.insert_gcroot_marker(mark) def target_arglocs(self, loop_token): return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(imm(loop_token._x86_loop_code)) + if loop_token is self.currently_compiling_loop: + curpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(self.looppos - curpos) + else: + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): size = max(size, self.cpu.gc_ll_descr.minimal_size_in_nursery) - self.mc.ensure_bytes_available(256) self.mc.MOV(eax, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (eax.value, size)) self.mc.CMP(edx, heap(nursery_top_adr)) @@ -1890,7 +1830,7 @@ offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.overwrite(jmp_adr-1, chr(offset)) # on 64-bits, 'tid' is a value that fits in 31 bits self.mc.MOV_mi((eax.value, 0), tid) self.mc.MOV(heap(nursery_free_adr), edx) Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/regalloc.py Fri Dec 3 11:04:29 2010 @@ -60,32 +60,6 @@ r15: 5, } -class FloatConstants(object): - BASE_CONSTANT_SIZE = 1000 - - def __init__(self): - self.cur_array_free = 0 - self.const_id = 0 - - def _get_new_array(self): - n = self.BASE_CONSTANT_SIZE - # known to leak - self.cur_array = lltype.malloc(rffi.CArray(lltype.Float), n, - flavor='raw', track_allocation=False) - self.cur_array_free = n - _get_new_array._dont_inline_ = True - - def record_float(self, floatval): - if self.cur_array_free == 0: - self._get_new_array() - arr = self.cur_array - n = self.cur_array_free - 1 - arr[n] = floatval - self.cur_array_free = n - self.const_id += 1 - return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) - - class X86XMMRegisterManager(RegisterManager): box_types = [FLOAT] @@ -93,20 +67,11 @@ # we never need lower byte I hope save_around_call_regs = all_regs - def __init__(self, longevity, frame_manager=None, assembler=None): - RegisterManager.__init__(self, longevity, frame_manager=frame_manager, - assembler=assembler) - if assembler is None: - self.float_constants = FloatConstants() - else: - if assembler._float_constants is None: - assembler._float_constants = FloatConstants() - self.float_constants = assembler._float_constants - def convert_to_imm(self, c): - const_id, adr = self.float_constants.record_float(c.getfloat()) - return ConstFloatLoc(adr, const_id) - + adr = self.assembler.datablockwrapper.malloc_aligned(8, 8) + rffi.cast(rffi.CArrayPtr(rffi.DOUBLE), adr)[0] = c.getfloat() + return ConstFloatLoc(adr) + def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately @@ -320,11 +285,22 @@ def locs_for_fail(self, guard_op): return [self.loc(v) for v in guard_op.getfailargs()] + def get_current_depth(self): + # return (self.fm.frame_depth, self.param_depth), but trying to share + # the resulting tuple among several calls + arg0 = self.fm.frame_depth + arg1 = self.param_depth + result = self.assembler._current_depths_cache + if result[0] != arg0 or result[1] != arg1: + result = (arg0, arg1) + self.assembler._current_depths_cache = result + return result + def perform_with_guard(self, op, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) self.rm.position += 1 self.xrm.position += 1 - current_depths = (self.fm.frame_depth, self.param_depth) + current_depths = self.get_current_depth() self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, current_depths) @@ -340,7 +316,7 @@ arglocs)) else: self.assembler.dump('%s(%s)' % (guard_op, arglocs)) - current_depths = (self.fm.frame_depth, self.param_depth) + current_depths = self.get_current_depth() self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, current_depths) @@ -1097,7 +1073,8 @@ if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert reg in self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX gcrootmap.add_callee_save_reg(shape, self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX[reg]) - return gcrootmap.compress_callshape(shape) + return gcrootmap.compress_callshape(shape, + self.assembler.datablockwrapper) def consider_force_token(self, op): loc = self.rm.force_allocate_reg(op.result) Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/regloc.py Fri Dec 3 11:04:29 2010 @@ -177,24 +177,15 @@ class ConstFloatLoc(AssemblerLocation): # XXX: We have to use this class instead of just AddressLoc because - # AddressLoc is "untyped" and also we to have need some sort of unique - # identifier that we can use in _getregkey (for jump.py) - + # we want a width of 8 (... I think. Check this!) _immutable_ = True - width = 8 - def __init__(self, address, const_id): + def __init__(self, address): self.value = address - self.const_id = const_id def __repr__(self): - return '' % (self.value, self.const_id) - - def _getregkey(self): - # XXX: 1000 is kind of magic: We just don't want to be confused - # with any registers - return 1000 + self.const_id + return '' % (self.value,) def location_code(self): return 'j' @@ -334,9 +325,9 @@ if code == possible_code: val = getattr(loc, "value_" + possible_code)() if possible_code == 'i': - offset = intmask(val - (self.tell() + 5)) - if rx86.fits_in_32bits(offset): + if self.WORD == 4: _rx86_getattr(self, name + "_l")(val) + self.add_pending_relocation() else: assert self.WORD == 8 self._load_scratch(val) Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/runner.py Fri Dec 3 11:04:29 2010 @@ -44,6 +44,7 @@ def setup_once(self): self.profile_agent.startup() + self.assembler.setup_once() def finish_once(self): self.assembler.finish_once() @@ -53,9 +54,12 @@ self.assembler.assemble_loop(inputargs, operations, looptoken, log=log) - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): + clt = original_loop_token.compiled_loop_token + clt.compiling_a_bridge() self.assembler.assemble_bridge(faildescr, inputargs, operations, - log=log) + original_loop_token, log=log) def set_future_value_int(self, index, intvalue): self.assembler.fail_boxes_int.setitem(index, intvalue) @@ -124,8 +128,8 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = -1 - bytecode = rffi.cast(rffi.UCHARP, - faildescr._x86_failure_recovery_bytecode) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) # start of "no gc operation!" block fail_index_2 = self.assembler.grab_frame_values( bytecode, @@ -165,17 +169,12 @@ CPU = CPU386 # silence warnings - -history.LoopToken._x86_param_depth = 0 -history.LoopToken._x86_arglocs = (None, None) -history.LoopToken._x86_frame_depth = 0 -history.LoopToken._x86_bootstrap_code = 0 -history.LoopToken._x86_direct_bootstrap_code = 0 -history.LoopToken._x86_failure_recovery_bytecode = 0 -history.LoopToken._x86_loop_code = 0 -history.LoopToken._x86_current_depths = (0, 0) - -compile._DoneWithThisFrameDescr._x86_current_depths = (0, 0) -compile._DoneWithThisFrameDescr._x86_failure_recovery_bytecode = 0 -compile._DoneWithThisFrameDescr._x86_adr_jump_offset = 0 - +##history.LoopToken._x86_param_depth = 0 +##history.LoopToken._x86_arglocs = (None, None) +##history.LoopToken._x86_frame_depth = 0 +##history.LoopToken._x86_bootstrap_code = 0 +##history.LoopToken._x86_direct_bootstrap_code = 0 +##history.LoopToken._x86_loop_code = 0 +##history.LoopToken._x86_debug_checksum = 0 +##compile.AbstractFailDescr._x86_current_depths = (0, 0) +##compile.AbstractFailDescr._x86_adr_jump_offset = 0 Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/rx86.py Fri Dec 3 11:04:29 2010 @@ -137,10 +137,9 @@ # ____________________________________________________________ # Emit an immediate displacement (relative to the cur insn) -def encode_relative(mc, target, _, orbyte): +def encode_relative(mc, relative_target, _, orbyte): assert orbyte == 0 - offset = intmask(target - (mc.tell() + 4)) - mc.writeimm32(offset) + mc.writeimm32(relative_target) return 0 def relative(argnum): Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/support.py Fri Dec 3 11:04:29 2010 @@ -1,4 +1,7 @@ +import sys from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from pypy.translator.tool.cbuild import ExternalCompilationInfo + def values_array(TP, size): ATP = lltype.GcArray(TP) @@ -23,3 +26,22 @@ return True return ValuesArray() + +# ____________________________________________________________ + +memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) + +# ____________________________________________________________ + +if sys.platform == 'win32': + ensure_sse2_floats = lambda : None +else: + _sse2_eci = ExternalCompilationInfo( + compile_extra = ['-msse2', '-mfpmath=sse'], + separate_module_sources = ['void PYPY_NO_OP(void) {}'], + ) + ensure_sse2_floats = rffi.llexternal('PYPY_NO_OP', [], lltype.Void, + compilation_info=_sse2_eci, + sandboxsafe=True) Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_assembler.py Fri Dec 3 11:04:29 2010 @@ -1,5 +1,5 @@ from pypy.jit.backend.x86.regloc import * -from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper +from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask @@ -19,28 +19,10 @@ return 42 class FakeMC: - def __init__(self, base_address=0): + def __init__(self): self.content = [] - self._size = 100 - self.base_address = base_address - def writechr(self, n): - self.content.append(n) - def tell(self): - return self.base_address + len(self.content) - def get_relative_pos(self): - return len(self.content) - def JMP(self, *args): - self.content.append(("JMP", args)) - def done(self): - pass - def PUSH_r(self, reg): - pass - def POP_r(self, reg): - pass - -class FakeAssembler: - def write_pending_failure_recoveries(self): - pass + def writechar(self, char): + self.content.append(ord(char)) def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -255,41 +237,3 @@ assert assembler.fail_boxes_int.getitem(i) == expected_ints[i] assert assembler.fail_boxes_ptr.getitem(i) == expected_ptrs[i] assert assembler.fail_boxes_float.getitem(i) == expected_floats[i] - -class FakeProfileAgent(object): - def __init__(self): - self.functions = [] - - def native_code_written(self, name, address, size): - self.functions.append((name, address, size)) - -class FakeMCWrapper(MachineCodeBlockWrapper): - count = 0 - def _instantiate_mc(self): - self.count += 1 - return FakeMC(200 * (self.count - 1)) - -def test_mc_wrapper_profile_agent(): - agent = FakeProfileAgent() - assembler = FakeAssembler() - mc = FakeMCWrapper(assembler, 100, agent) - mc.start_function("abc") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4)] - mc.writechr("x") - mc.start_function("cde") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4), ("cde", 5, 4)] - mc.start_function("xyz") - for i in range(50): - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4), ("cde", 5, 4), ("xyz", 9, 29), ("xyz", 200, 22)] Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_gc_integration.py Fri Dec 3 11:04:29 2010 @@ -33,7 +33,8 @@ def add_callee_save_reg(self, shape, reg_index): index_to_name = { 1: 'ebx', 2: 'esi', 3: 'edi' } shape.append(index_to_name[reg_index]) - def compress_callshape(self, shape): + def compress_callshape(self, shape, datablockwrapper): + assert datablockwrapper == 'fakedatablockwrapper' assert shape[0] == 'shape' return ['compressed'] + shape[1:] @@ -58,7 +59,9 @@ def test_mark_gc_roots(self): cpu = CPU(None, None) + cpu.setup_once() regalloc = RegAlloc(MockAssembler(cpu, MockGcDescr(False))) + regalloc.assembler.datablockwrapper = 'fakedatablockwrapper' boxes = [BoxPtr() for i in range(len(X86RegisterManager.all_regs))] longevity = {} for box in boxes: @@ -90,6 +93,7 @@ cpu = CPU(None, None) cpu.gc_ll_descr = MockGcDescr(False) + cpu.setup_once() S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', ('field', lltype.Ptr(S)), @@ -214,6 +218,7 @@ cpu = CPU(None, None) cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() + cpu.setup_once() NODE = lltype.Struct('node', ('tid', lltype.Signed), ('value', lltype.Signed)) Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regalloc.py Fri Dec 3 11:04:29 2010 @@ -9,7 +9,7 @@ from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ - FloatConstants, is_comparison_or_ovf_op + is_comparison_or_ovf_op from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -41,7 +41,10 @@ self.movs = [] self.performs = [] self.lea = [] - self.cpu = cpu or CPU(None, None) + if cpu is None: + cpu = CPU(None, None) + cpu.setup_once() + self.cpu = cpu if gc_ll_descr is None: gc_ll_descr = MockGcDescr(False) self.cpu.gc_ll_descr = gc_ll_descr @@ -76,6 +79,7 @@ class BaseTestRegalloc(object): cpu = CPU(None, None) + cpu.setup_once() def raising_func(i): if i: @@ -166,7 +170,8 @@ assert ([box.type for box in bridge.inputargs] == [box.type for box in guard_op.getfailargs()]) faildescr = guard_op.getdescr() - self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations) + self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations, + loop.token) return bridge def run(self, loop): @@ -515,16 +520,6 @@ self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] - def test_float_overflow_const_list(self): - ops = ['[f0]'] - BASE_CONSTANT_SIZE = FloatConstants.BASE_CONSTANT_SIZE - for i in range(BASE_CONSTANT_SIZE * 2): - ops.append('f%d = float_add(f%d, 3.5)' % (i + 1, i)) - ops.append('finish(f%d)' % (BASE_CONSTANT_SIZE * 2)) - ops = "\n".join(ops) - self.interpret(ops, [0.1]) - assert abs(self.getfloat(0) - (BASE_CONSTANT_SIZE * 2) * 3.5 - 0.1) < 0.00001 - def test_lt_const(self): ops = ''' [f0] Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regalloc2.py Fri Dec 3 11:04:29 2010 @@ -19,6 +19,7 @@ ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, 9) @@ -41,6 +42,7 @@ ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, -10) @@ -137,6 +139,7 @@ ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, -13) @@ -251,6 +254,7 @@ ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, 17) Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_regloc.py Fri Dec 3 11:04:29 2010 @@ -1,4 +1,4 @@ -import struct +import struct, sys from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.test.test_rx86 import CodeBuilder32, CodeBuilder64, assert_encodes_as from pypy.jit.backend.x86.assembler import heap @@ -37,26 +37,36 @@ assert_encodes_as(cb64, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') assert_encodes_as(cb64, "CMP16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\x81\x7D\x00\x39\x30') -def test_jmp_wraparound(): - if not IS_X86_32: - py.test.skip() - - pos_addr = intmask(0x7FFFFF00) - neg_addr = intmask(0x800000BB) - - # JMP to "negative" address from "positive" address - s = cb32() - s.base_address = pos_addr - s.JMP(ImmedLoc(neg_addr)) - expected_ofs = neg_addr - (pos_addr+5) - assert s.getvalue() == '\xE9' + struct.pack(" sys.maxint: + continue + mc = codebuf.MachineCodeBlockWrapper() + mc.CALL(ImmedLoc(target)) + length = mc.get_relative_pos() + buf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw') + rawstart = rffi.cast(lltype.Signed, buf) + if IS_X86_32: + assert length == 5 + assert mc.relocations == [5] + expected = "\xE8" + struct.pack('= 40 # randomish number @@ -370,7 +376,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" # Would be exactly ==, but there are some guard failure recovery @@ -397,104 +403,15 @@ assert res.value == 4.0 -class TestX86OverflowMC(TestX86): - - def setup_method(self, meth): - self.cpu = CPU(rtyper=None, stats=FakeStats()) - self.cpu.assembler.mc_size = 1024 - - def test_overflow_mc(self): - ops = [] - base_v = BoxInt() - v = base_v - for i in range(1024): - next_v = BoxInt() - ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) - v = next_v - ops.append(ResOperation(rop.FINISH, [v], None, - descr=BasicFailDescr())) - looptoken = LoopToken() - self.cpu.assembler.setup() - old_mc_mc = self.cpu.assembler.mc._mc - self.cpu.compile_loop([base_v], ops, looptoken) - assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed - self.cpu.set_future_value_int(0, base_v.value) - self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 1024 - - def test_overflow_guard_float_cmp(self): - # The float comparisons on x86 tend to use small relative jumps, - # which may run into trouble if they fall on the edge of a - # MachineCodeBlock change. - a = BoxFloat(1.0) - b = BoxFloat(2.0) - failed = BoxInt(41) - finished = BoxInt(42) - - # We select guards that will always succeed, so that execution will - # continue through the entire set of comparisions - ops_to_test = ( - (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), - - (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), - - (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), - - (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), - - (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), - (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), - - (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), - (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), - ) - - for float_op, args, guard_op in ops_to_test: - ops = [] - - for i in range(200): - cmp_result = BoxInt() - ops.append(ResOperation(float_op, args, cmp_result)) - ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) - ops[-1].setfailargs([failed]) - - ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) - - looptoken = LoopToken() - self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) - self.cpu.set_future_value_float(0, a.value) - self.cpu.set_future_value_float(1, b.value) - self.cpu.set_future_value_int(2, failed.value) - self.cpu.set_future_value_int(3, finished.value) - self.cpu.execute_token(looptoken) - - # Really just a sanity check. We're actually interested in - # whether the test segfaults. - assert self.cpu.get_latest_value_int(0) == finished.value - - def test_overflow_guard_exception(self): - for i in range(50): - self.test_exceptions() - - class TestDebuggingAssembler(object): def setup_method(self, meth): - self.pypylog = os.environ.get('PYPYLOG', None) - self.logfile = str(udir.join('x86_runner.log')) - os.environ['PYPYLOG'] = "mumble:" + self.logfile self.cpu = CPU(rtyper=None, stats=FakeStats()) - - def teardown_method(self, meth): - if self.pypylog is not None: - os.environ['PYPYLOG'] = self.pypylog + self.cpu.setup_once() def test_debugger_on(self): + from pypy.tool.logparser import parse_log_file, extract_category + from pypy.rlib import debug + loop = """ [i0] debug_merge_point('xyz', 0) @@ -504,17 +421,19 @@ jump(i1) """ ops = parse(loop) - self.cpu.assembler.set_debug(True) - self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) - self.cpu.set_future_value_int(0, 0) - self.cpu.execute_token(ops.token) - # check debugging info - name, struct = self.cpu.assembler.loop_run_counters[0] - assert name == 0 # 'xyz' - assert struct.i == 10 - self.cpu.finish_once() - lines = py.path.local(self.logfile + ".count").readlines() - assert lines[0] == '0:10\n' # '10 xyz\n' + debug._log = dlog = debug.DebugLog() + try: + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + struct = self.cpu.assembler.loop_run_counters[0] + assert struct.i == 10 + self.cpu.finish_once() + finally: + debug._log = None + assert ('jit-backend-counts', [('debug_print', '0:10')]) in dlog def test_debugger_checksum(self): loop = """ Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_rx86.py Fri Dec 3 11:04:29 2010 @@ -5,7 +5,6 @@ class CodeBuilderMixin(object): def __init__(self): self.buffer = [] - self.base_address = 0x76543210 def writechar(self, c): assert isinstance(c, str) and len(c) == 1 @@ -14,9 +13,6 @@ def getvalue(self): return ''.join(self.buffer) - def tell(self): - return self.base_address + len(self.buffer) - def assert_encodes_as(code_builder_cls, insn_name, args, expected_encoding): s = code_builder_cls() getattr(s, insn_name)(*args) @@ -104,21 +100,18 @@ def test_call_l(s=None): s = s or CodeBuilder32() - s.CALL_l(0x01234567) - ofs = 0x01234567 - (0x76543210+5) - assert s.getvalue() == '\xE8' + struct.pack(" 5: + x = A1 + else: + x = A2 + x() + rtyper = support.annotate(f, [35]) + maingraph = rtyper.annotator.translator.graphs[0] + cw = CodeWriter(FakeCPU(rtyper), [FakeJitDriverSD(maingraph)]) + cw.find_all_graphs(MyFakePolicy()) + cw.make_jitcodes(verbose=True) + # + names = [jitcode.name for jitcode in cw.assembler.indirectcalltargets] + assert len(names) == 1 + assert names[0].startswith('instantiate_') and names[0].endswith('A1') + # + print cw.assembler.list_of_addr2name + names = dict.fromkeys([value + for key, value in cw.assembler.list_of_addr2name]) + assert 'A1' in names + assert 'A2' in names + def test_int_abs(): def f(n): return abs(n) Modified: pypy/branch/jitypes2/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/jitypes2/pypy/jit/codewriter/test/test_list.py Fri Dec 3 11:04:29 2010 @@ -56,9 +56,8 @@ if '/' in oopspec_name: oopspec_name, property = oopspec_name.split('/') def force_flags(op): - if property == 'NONNEG': return True, False - if property == 'NEG': return False, False - if property == 'CANRAISE': return False, True + if property == 'NONNEG': return True + if property == 'NEG': return False raise ValueError(property) tr._get_list_nonneg_canraise_flags = force_flags op = SpaceOperation('direct_call', @@ -122,9 +121,6 @@ check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_i %r0, , %i1 -> %i2 """) - builtin_test('list.getitem/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -139,9 +135,6 @@ check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_pure_i %r0, , %i1 -> %i2 """) - builtin_test('list.getitem_foldable/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -158,10 +151,6 @@ check_neg_index %r0, , %i0 -> %i1 setarrayitem_gc_i %r0, , %i1, %i2 """) - builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), - varoftype(lltype.Signed), - varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -206,9 +195,6 @@ check_resizable_neg_index %r0, , %i0 -> %i1 getlistitem_gc_i %r0, , , %i1 -> %i2 """) - builtin_test('list.getitem/CANRAISE', - [varoftype(VARLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_resizable_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), @@ -225,10 +211,6 @@ check_resizable_neg_index %r0, , %i0 -> %i1 setlistitem_gc_i %r0, , , %i1, %i2 """) - builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), - varoftype(lltype.Signed), - varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_resizable_len(): builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, Modified: pypy/branch/jitypes2/pypy/jit/codewriter/test/test_void_list.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/codewriter/test/test_void_list.py (original) +++ pypy/branch/jitypes2/pypy/jit/codewriter/test/test_void_list.py Fri Dec 3 11:04:29 2010 @@ -51,9 +51,6 @@ builtin_test('list.getitem/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -62,9 +59,6 @@ builtin_test('list.getitem_foldable/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem_foldable/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -75,10 +69,6 @@ varoftype(lltype.Signed), varoftype(lltype.Void)], lltype.Void, "") - builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), - varoftype(lltype.Signed), - varoftype(lltype.Void)], - lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -115,9 +105,6 @@ builtin_test('list.getitem/NEG', [varoftype(VARLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem/CANRAISE', - [varoftype(VARLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_resizable_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), @@ -128,10 +115,6 @@ varoftype(lltype.Signed), varoftype(lltype.Void)], lltype.Void, "") - builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), - varoftype(lltype.Signed), - varoftype(lltype.Void)], - lltype.Void, NotSupported) def test_resizable_len(): builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, Modified: pypy/branch/jitypes2/pypy/jit/conftest.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/conftest.py (original) +++ pypy/branch/jitypes2/pypy/jit/conftest.py Fri Dec 3 11:04:29 2010 @@ -1,7 +1,5 @@ import py -option = py.test.config.option - def pytest_addoption(parser): group = parser.getgroup("JIT options") group.addoption('--slow', action="store_true", Modified: pypy/branch/jitypes2/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/compile.py Fri Dec 3 11:04:29 2010 @@ -1,4 +1,4 @@ - +import weakref from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype from pypy.objspace.flow.model import Constant, Variable @@ -48,6 +48,32 @@ loop_token.outermost_jitdriver_sd = jitdriver_sd return loop_token +def record_loop_or_bridge(loop): + """Do post-backend recordings and cleanups on 'loop'. + """ + # get the original loop token (corresponding to 'loop', or if that is + # a bridge, to the loop that this bridge belongs to) + looptoken = loop.token + assert looptoken is not None + wref = weakref.ref(looptoken) + for op in loop.operations: + descr = op.getdescr() + if isinstance(descr, ResumeDescr): + descr.wref_original_loop_token = wref # stick it there + n = descr.index + if n >= 0: # we also record the resumedescr number + looptoken.compiled_loop_token.record_faildescr_index(n) + elif isinstance(descr, LoopToken): + # for a JUMP or a CALL_ASSEMBLER: record it as a potential jump. + # (the following test is not enough to prevent more complicated + # cases of cycles, but at least it helps in simple tests of + # test_memgr.py) + if descr is not looptoken: + looptoken.record_jump_to(descr) + op.setdescr(None) # clear reference, mostly for tests + # mostly for tests: make sure we don't keep a reference to the LoopToken + loop.token = None + # ____________________________________________________________ def compile_new_loop(metainterp, old_loop_tokens, start): @@ -77,6 +103,7 @@ return old_loop_token send_loop_to_backend(metainterp_sd, loop, "loop") insert_loop_token(old_loop_tokens, loop_token) + record_loop_or_bridge(loop) return loop_token def insert_loop_token(old_loop_tokens, loop_token): @@ -117,8 +144,11 @@ else: loop._ignore_during_counting = True metainterp_sd.log("compiled new " + type) + if metainterp_sd.warmrunnerdesc is not None: # for tests + metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(loop.token) -def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): +def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations, + original_loop_token): n = metainterp_sd.cpu.get_fail_descr_number(faildescr) metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) if not we_are_translated(): @@ -127,7 +157,8 @@ metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: - metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, + original_loop_token) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() @@ -227,9 +258,6 @@ CNT_FLOAT = -0x60000000 CNT_MASK = 0x1FFFFFFF - def __init__(self, metainterp_sd): - self.metainterp_sd = metainterp_sd - def store_final_boxes(self, guard_op, boxes): guard_op.setfailargs(boxes) self.guard_opnum = guard_op.getopnum() @@ -314,12 +342,14 @@ def compile_and_attach(self, metainterp, new_loop): # We managed to create a bridge. Attach the new operations - # to the corrsponding guard_op and compile from there + # to the corresponding guard_op and compile from there + assert metainterp.resumekey_original_loop_token is not None + new_loop.token = metainterp.resumekey_original_loop_token inputargs = metainterp.history.inputargs if not we_are_translated(): self._debug_suboperations = new_loop.operations send_bridge_to_backend(metainterp.staticdata, self, inputargs, - new_loop.operations) + new_loop.operations, new_loop.token) def copy_all_attrbutes_into(self, res): # XXX a bit ugly to have to list them all here @@ -331,14 +361,14 @@ res.rd_pendingfields = self.rd_pendingfields def _clone_if_mutable(self): - res = ResumeGuardDescr(self.metainterp_sd) + res = ResumeGuardDescr() self.copy_all_attrbutes_into(res) return res class ResumeGuardForcedDescr(ResumeGuardDescr): def __init__(self, metainterp_sd, jitdriver_sd): - ResumeGuardDescr.__init__(self, metainterp_sd) + self.metainterp_sd = metainterp_sd self.jitdriver_sd = jitdriver_sd def handle_fail(self, metainterp_sd, jitdriver_sd): @@ -482,6 +512,8 @@ metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd redargs = new_loop.inputargs + # We make a new LoopToken for this entry bridge, and stick it + # to every guard in the loop. new_loop_token = make_loop_token(len(redargs), jitdriver_sd) new_loop.token = new_loop_token send_loop_to_backend(metainterp_sd, new_loop, "entry bridge") @@ -489,12 +521,14 @@ jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( self.original_greenkey, new_loop_token) - # store the new loop in compiled_merge_points too + # store the new loop in compiled_merge_points_wref too old_loop_tokens = metainterp.get_compiled_merge_points( self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token old_loop_tokens.append(new_loop_token) + metainterp.set_compiled_merge_points(self.original_greenkey, + old_loop_tokens) def reset_counter_from_failure(self): pass @@ -529,6 +563,7 @@ # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr) prepare_last_operation(new_loop, target_loop_token) resumekey.compile_and_attach(metainterp, new_loop) + record_loop_or_bridge(new_loop) return target_loop_token def prepare_last_operation(new_loop, target_loop_token): @@ -555,7 +590,8 @@ propagate_exception_descr = PropagateExceptionDescr() -def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redboxes): +def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redboxes, + memory_manager=None): """Make a LoopToken that corresponds to assembler code that just calls back the interpreter. Used temporarily: a fully compiled version of the code may end up replacing it. @@ -595,4 +631,6 @@ ] operations[1].setfailargs([]) cpu.compile_loop(inputargs, operations, loop_token, log=False) + if memory_manager is not None: # for tests + memory_manager.keep_loop_alive(loop_token) return loop_token Modified: pypy/branch/jitypes2/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/history.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/history.py Fri Dec 3 11:04:29 2010 @@ -4,8 +4,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic from pypy.rlib.objectmodel import compute_hash, compute_unique_id -from pypy.rlib.rarithmetic import intmask -from pypy.tool.uid import uid +from pypy.rlib.rarithmetic import intmask, r_int64 from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -730,14 +729,28 @@ outermost_jitdriver_sd = None # specnodes = ... # and more data specified by the backend when the loop is compiled - number = 0 + number = -1 + generation = r_int64(0) + # one purpose of LoopToken is to keep alive the CompiledLoopToken + # returned by the backend. When the LoopToken goes away, the + # CompiledLoopToken has its __del__ called, which frees the assembler + # memory and the ResumeGuards. + compiled_loop_token = None - def __init__(self, number=0): - self.number = number + def __init__(self): + # For memory management of assembled loops + self._keepalive_target_looktokens = {} # set of other LoopTokens + + def record_jump_to(self, target_loop_token): + self._keepalive_target_looktokens[target_loop_token] = None + + def __repr__(self): + return '' % (self.number, self.generation) def repr_of_descr(self): return '' % self.number + class TreeLoop(object): inputargs = None operations = None Modified: pypy/branch/jitypes2/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/jitprof.py Fri Dec 3 11:04:29 2010 @@ -2,9 +2,9 @@ """ A small helper module for profiling JIT """ -import os import time -from pypy.rlib.debug import debug_print +from pypy.rlib.debug import debug_print, debug_start, debug_stop +from pypy.rlib.debug import have_debug_prints from pypy.jit.metainterp.jitexc import JitException counters=""" @@ -24,6 +24,10 @@ NVIRTUALS NVHOLES NVREUSED +TOTAL_COMPILED_LOOPS +TOTAL_COMPILED_BRIDGES +TOTAL_FREED_LOOPS +TOTAL_FREED_BRIDGES """ def _setup(): @@ -35,6 +39,7 @@ _setup() JITPROF_LINES = ncounters + 1 + 1 # one for TOTAL, 1 for calls, update if needed +_CPU_LINES = 4 # the last 4 lines are stored on the cpu class BaseProfiler(object): pass @@ -48,9 +53,6 @@ def finish(self): pass - def set_printing(self, printing): - pass - def start_tracing(self): pass @@ -90,23 +92,19 @@ counters = None calls = 0 current = None - printing = True + cpu = None def start(self): self.starttime = self.timer() self.t1 = self.starttime self.times = [0, 0] - self.counters = [0] * ncounters + self.counters = [0] * (ncounters - _CPU_LINES) self.calls = 0 self.current = [] def finish(self): self.tk = self.timer() - if self.printing: - self.print_stats() - - def set_printing(self, printing): - self.printing = printing + self.print_stats() def _start(self, event): t0 = self.t1 @@ -154,6 +152,12 @@ self.calls += 1 def print_stats(self): + debug_start("jit-summary") + if have_debug_prints(): + self._print_stats() + debug_stop("jit-summary") + + def _print_stats(self): cnt = self.counters tim = self.times calls = self.calls @@ -161,8 +165,8 @@ self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) self._print_intline("Running asm", cnt[RUNNING]) self._print_intline("Blackhole", cnt[BLACKHOLE]) - line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) - os.write(2, line) + line = "TOTAL: \t\t%f" % (self.tk - self.starttime, ) + debug_print(line) self._print_intline("ops", cnt[OPS]) self._print_intline("recorded ops", cnt[RECORDED_OPS]) self._print_intline(" calls", calls) @@ -176,17 +180,26 @@ self._print_intline("nvirtuals", cnt[NVIRTUALS]) self._print_intline("nvholes", cnt[NVHOLES]) self._print_intline("nvreused", cnt[NVREUSED]) + cpu = self.cpu + if cpu is not None: # for some tests + self._print_intline("Total # of loops", + cpu.total_compiled_loops) + self._print_intline("Total # of bridges", + cpu.total_compiled_bridges) + self._print_intline("Freed # of loops", + cpu.total_freed_loops) + self._print_intline("Freed # of bridges", + cpu.total_freed_bridges) def _print_line_time(self, string, i, tim): - final = "%s:%s\t%d\t%f\n" % (string, " " * max(0, 13-len(string)), i, tim) - os.write(2, final) + final = "%s:%s\t%d\t%f" % (string, " " * max(0, 13-len(string)), i, tim) + debug_print(final) def _print_intline(self, string, i): final = string + ':' + " " * max(0, 16-len(string)) - final += '\t' + str(i) + '\n' - os.write(2, final) - - + final += '\t' + str(i) + debug_print(final) + class BrokenProfilerData(JitException): pass Modified: pypy/branch/jitypes2/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/pyjitpl.py Fri Dec 3 11:04:29 2010 @@ -14,11 +14,10 @@ from pypy.jit.metainterp.logger import Logger from pypy.jit.metainterp.jitprof import EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE -from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG +from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE from pypy.jit.metainterp.jitexc import JitException, get_llexception from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize -from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr from pypy.jit.codewriter import heaptracker @@ -613,8 +612,10 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) arrayindex = vinfo.array_field_by_descrs[arrayfielddescr] index = indexbox.getint() - if index < 0: - index += vinfo.get_array_length(virtualizable, arrayindex) + # Support for negative index: disabled + # (see codewriter/jtransform.py, _check_no_vable_array). + #if index < 0: + # index += vinfo.get_array_length(virtualizable, arrayindex) assert 0 <= index < vinfo.get_array_length(virtualizable, arrayindex) return vinfo.get_index_in_array(virtualizable, arrayindex, index) @@ -1069,7 +1070,7 @@ resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, metainterp.jitdriver_sd) else: - resumedescr = compile.ResumeGuardDescr(metainterp_sd) + resumedescr = compile.ResumeGuardDescr() guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None @@ -1223,6 +1224,7 @@ self.logger_ops = Logger(self, guard_number=True) self.profiler = ProfilerClass() + self.profiler.cpu = cpu self.warmrunnerdesc = warmrunnerdesc backendmodule = self.cpu.__module__ @@ -1341,6 +1343,11 @@ return jitcode return None + def try_to_free_some_loops(self): + # Increase here the generation recorded by the memory manager. + if self.warmrunnerdesc is not None: # for tests + self.warmrunnerdesc.memory_manager.next_generation() + # ---------------- logging ------------------------ def log(self, msg): @@ -1637,6 +1644,7 @@ self.staticdata._setup_once() self.staticdata.profiler.start_tracing() assert jitdriver_sd is self.jitdriver_sd + self.staticdata.try_to_free_some_loops() self.create_empty_history() try: original_boxes = self.initialize_original_boxes(jitdriver_sd, *args) @@ -1665,10 +1673,15 @@ debug_start('jit-tracing') self.staticdata.profiler.start_tracing() assert isinstance(key, compile.ResumeGuardDescr) + # store the resumekey.wref_original_loop_token() on 'self' to make + # sure that it stays alive as long as this MetaInterp + self.resumekey_original_loop_token = key.wref_original_loop_token() + self.staticdata.try_to_free_some_loops() self.initialize_state_from_guard_failure(key) try: return self._handle_guard_failure(key) finally: + self.resumekey_original_loop_token = None self.staticdata.profiler.end_tracing() debug_stop('jit-tracing') @@ -1678,6 +1691,8 @@ self.seen_loop_header_for_jdindex = -1 try: self.prepare_resume_from_failure(key.guard_opnum) + if self.resumekey_original_loop_token is None: # very rare case + raise SwitchToBlackhole(ABORT_BRIDGE) self.interpret() except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) @@ -1799,10 +1814,15 @@ raise NotImplementedError(opname[opnum]) def get_compiled_merge_points(self, greenkey): + """Get the list of looptokens corresponding to the greenkey. + Turns the (internal) list of weakrefs into regular refs. + """ + cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) + return cell.get_compiled_merge_points() + + def set_compiled_merge_points(self, greenkey, looptokens): cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) - if cell.compiled_merge_points is None: - cell.compiled_merge_points = [] - return cell.compiled_merge_points + cell.set_compiled_merge_points(looptokens) def compile(self, original_boxes, live_arg_boxes, start): num_green_args = self.jitdriver_sd.num_green_args @@ -1812,6 +1832,7 @@ self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) loop_token = compile.compile_new_loop(self, old_loop_tokens, start) if loop_token is not None: # raise if it *worked* correctly + self.set_compiled_merge_points(greenkey, old_loop_tokens) raise GenerateMergePoint(live_arg_boxes, loop_token) self.history.operations.pop() # remove the JUMP Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_basic.py Fri Dec 3 11:04:29 2010 @@ -19,7 +19,11 @@ from pypy.jit.metainterp import simple_optimize class FakeJitCell: - compiled_merge_points = None + __compiled_merge_points = [] + def get_compiled_merge_points(self): + return self.__compiled_merge_points[:] + def set_compiled_merge_points(self, lst): + self.__compiled_merge_points = lst class FakeWarmRunnerState: def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): @@ -35,7 +39,6 @@ optimize_bridge = staticmethod(simple_optimize.optimize_bridge) trace_limit = sys.maxint - debug_level = 2 func._jit_unroll_safe_ = True rtyper = support.annotate(func, values, type_system=type_system) @@ -1698,6 +1701,29 @@ res = self.meta_interp(f, [5, 2]) assert 4 < res < 14 + def test_compute_identity_hash(self): + from pypy.rlib.objectmodel import compute_identity_hash + class A(object): + pass + def f(): + a = A() + return compute_identity_hash(a) == compute_identity_hash(a) + res = self.interp_operations(f, []) + assert res + # a "did not crash" kind of test + + def test_compute_unique_id(self): + from pypy.rlib.objectmodel import compute_unique_id + class A(object): + pass + def f(): + a1 = A() + a2 = A() + return (compute_unique_id(a1) == compute_unique_id(a1) and + compute_unique_id(a1) != compute_unique_id(a2)) + res = self.interp_operations(f, []) + assert res + class TestOOtype(BasicTests, OOJitMixin): Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_compile.py Fri Dec 3 11:04:29 2010 @@ -42,7 +42,6 @@ class FakeState: optimize_loop = staticmethod(optimize.optimize_loop) - debug_level = 0 class FakeGlobalData: loopnumbering = 0 @@ -54,6 +53,7 @@ stats = Stats() profiler = jitprof.EmptyProfiler() + warmrunnerdesc = None def log(self, msg, event_kind=None): pass @@ -207,14 +207,12 @@ class ExitFrameWithExceptionRef(Exception): pass FakeMetaInterpSD.cpu = cpu - class FakeJitDriverSD: - pass cpu.set_future_value_int(0, -156) cpu.set_future_value_int(1, -178) cpu.set_future_value_int(2, -190) fail_descr = cpu.execute_token(loop_token) try: - fail_descr.handle_fail(FakeMetaInterpSD(), FakeJitDriverSD()) + fail_descr.handle_fail(FakeMetaInterpSD(), None) except FakeMetaInterpSD.ExitFrameWithExceptionRef, e: assert lltype.cast_opaque_ptr(lltype.Ptr(EXC), e.args[1]) == llexc else: Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_list.py Fri Dec 3 11:04:29 2010 @@ -188,6 +188,26 @@ assert res == f(4) self.check_loops(call=0, getfield_gc=0) + def test_fold_indexerror(self): + jitdriver = JitDriver(greens = [], reds = ['total', 'n', 'lst']) + def f(n): + lst = [] + total = 0 + while n > 0: + jitdriver.can_enter_jit(lst=lst, n=n, total=total) + jitdriver.jit_merge_point(lst=lst, n=n, total=total) + lst.append(n) + try: + total += lst[n] + except IndexError: + total += 1000 + n -= 1 + return total + + res = self.meta_interp(f, [15], listops=True) + assert res == f(15) + self.check_loops(guard_exception=0) + class TestOOtype(ListTests, OOJitMixin): pass Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_logger.py Fri Dec 3 11:04:29 2010 @@ -14,11 +14,20 @@ def capturing(func, *args, **kwds): log_stream = StringIO() - debug._stderr = log_stream + class MyDebugLog: + def debug_print(self, *args): + for arg in args: + print >> log_stream, arg, + print >> log_stream + def debug_start(self, *args): + pass + def debug_stop(self, *args): + pass try: + debug._log = MyDebugLog() func(*args, **kwds) finally: - debug._stderr = sys.stderr + debug._log = None return log_stream.getvalue() class Logger(logger.Logger): @@ -112,7 +121,8 @@ equaloplists(loop.operations, oloop.operations) def test_jump(self): - namespace = {'target': LoopToken(3)} + namespace = {'target': LoopToken()} + namespace['target'].number = 3 inp = ''' [i0] jump(i0, descr=target) Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizeopt.py Fri Dec 3 11:04:29 2010 @@ -41,7 +41,7 @@ b1 = BoxInt() opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu), None) - fdescr = ResumeGuardDescr(None) + fdescr = ResumeGuardDescr() op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr) # setup rd data fi0 = resume.FrameInfo(None, "code0", 11) Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_pyjitpl.py Fri Dec 3 11:04:29 2010 @@ -17,6 +17,7 @@ portal.setup(None) class FakeStaticData: cpu = None + warmrunnerdesc = None metainterp = pyjitpl.MetaInterp(FakeStaticData(), None) metainterp.framestack = [] @@ -53,6 +54,7 @@ def test_remove_consts_and_duplicates(): class FakeStaticData: cpu = None + warmrunnerdesc = None def is_another_box_like(box, referencebox): assert box is not referencebox assert isinstance(box, referencebox.clonebox().__class__) Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_recursive.py Fri Dec 3 11:04:29 2010 @@ -927,12 +927,16 @@ x=x) frame.s = hint(frame.s, promote=True) n -= 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] frame.s += 1 if codeno == 0: subframe = Frame([n, n+1, n+2, n+3], 0) x += f(1, 10, 1, subframe) - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] x += len(frame.l) frame.s -= 1 return x Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_virtualizable.py Fri Dec 3 11:04:29 2010 @@ -480,9 +480,13 @@ myjitdriver.jit_merge_point(frame=frame, n=n, x=x) frame.s = hint(frame.s, promote=True) n -= 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] frame.s += 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] x += len(frame.l) frame.s -= 1 return x @@ -994,7 +998,9 @@ jitdriver.can_enter_jit(frame=frame, n=n) jitdriver.jit_merge_point(frame=frame, n=n) popped = frame.stack[frame.stackpos] - frame.stackpos -= 1 + sp = frame.stackpos - 1 + assert sp >= 0 + frame.stackpos = sp to_push = intmask(popped * 3) frame.stack[frame.stackpos] = to_push frame.stackpos += 1 Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_warmspot.py Fri Dec 3 11:04:29 2010 @@ -132,91 +132,6 @@ assert state.optimize_loop is optimize.optimize_loop assert state.optimize_bridge is optimize.optimize_bridge - def test_static_debug_level(self, capfd): - py.test.skip("debug_level is being deprecated") - from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS - from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler - - myjitdriver = JitDriver(greens = [], reds = ['n']) - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - return n - - capfd.readouterr() - self.meta_interp(f, [10], debug_level=DEBUG_OFF, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not "Running asm" in err - self.meta_interp(f, [10], debug_level=DEBUG_PROFILE, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not 'compiled new' in err - assert "Running asm" in err - - self.meta_interp(f, [10], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert 'ENTER' in err - assert 'LEAVE' in err - assert "Running asm" in err - - self.meta_interp(f, [10], debug_level=DEBUG_STEPS, - ProfilerClass=EmptyProfiler) - out, err = capfd.readouterr() - assert 'ENTER' in err - assert 'LEAVE' in err - assert not "Running asm" in err - - def test_set_param_debug(self): - py.test.skip("debug_level is being deprecated") - from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS - from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler - - myjitdriver = JitDriver(greens = [], reds = ['n']) - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - return n - - def main(n, debug): - myjitdriver.set_param("debug", debug) - print f(n) - - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_OFF], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not "Running asm" in err - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_PROFILE], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not 'compiled new' in err - assert "Running asm" in err - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_STEPS], debug_level=DEBUG_OFF, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert 'ENTER' in err - assert 'LEAVE' in err - assert "Running asm" in err - def test_unwanted_loops(self): mydriver = JitDriver(reds = ['n', 'total', 'm'], greens = []) @@ -378,23 +293,30 @@ exc_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) cls.exc_vtable = exc_vtable - class FakeFailDescr(object): + class FakeLoopToken: def __init__(self, no): self.no = no + self.generation = 0 + + class FakeFailDescr(object): + def __init__(self, looptoken): + assert isinstance(looptoken, FakeLoopToken) + self.looptoken = looptoken def handle_fail(self, metainterp_sd, jitdrivers_sd): - if self.no == 0: + no = self.looptoken.no + if no == 0: raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3) - if self.no == 1: + if no == 1: raise metainterp_sd.warmrunnerdesc.ContinueRunningNormally( [0], [], [], [1], [], []) - if self.no == 3: + if no == 3: exc = lltype.malloc(OBJECT) exc.typeptr = exc_vtable raise metainterp_sd.warmrunnerdesc.ExitFrameWithExceptionRef( metainterp_sd.cpu, lltype.cast_opaque_ptr(llmemory.GCREF, exc)) - return self.no + return self.looptoken class FakeDescr: def as_vtable_size_descr(self): @@ -419,11 +341,11 @@ sizeof = nodescr def get_fail_descr_from_number(self, no): - return FakeFailDescr(no) + return FakeFailDescr(FakeLoopToken(no)) def execute_token(self, token): - assert token == 2 - return FakeFailDescr(1) + assert token.no == 2 + return FakeFailDescr(FakeLoopToken(1)) driver = JitDriver(reds = ['red'], greens = ['green']) Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_warmstate.py Fri Dec 3 11:04:29 2010 @@ -99,6 +99,8 @@ lltype.Float], lltype.Void)) class FakeWarmRunnerDesc: rtyper = FakeRTyper() + cpu = None + memory_manager = None class FakeJitDriverSD: _get_jitcell_at_ptr = llhelper(GETTER, getter) _set_jitcell_at_ptr = llhelper(SETTER, setter) @@ -126,6 +128,7 @@ future_values[j] = "float", value class FakeWarmRunnerDesc: cpu = FakeCPU() + memory_manager = None class FakeJitDriverSD: _red_args_types = ["int", "float"] virtualizable_info = None @@ -154,16 +157,20 @@ _get_jitcell_at_ptr = None state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() + class FakeLoopToken(object): + pass + looptoken = FakeLoopToken() state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], - "entry loop token") + looptoken) cell1 = get_jitcell(True, 5, 2.25) assert cell1.counter < 0 - assert cell1.entry_loop_token == "entry loop token" + assert cell1.get_entry_loop_token() is looptoken def test_make_jitdriver_callbacks_1(): class FakeWarmRunnerDesc: cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None @@ -189,6 +196,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) @@ -211,6 +219,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None @@ -233,6 +242,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None Modified: pypy/branch/jitypes2/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/warmspot.py Fri Dec 3 11:04:29 2010 @@ -12,11 +12,12 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.debug import debug_print, fatalerror +from pypy.rlib.debug import debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.unsimplify import call_final_function -from pypy.jit.metainterp import history, pyjitpl, gc +from pypy.jit.metainterp import history, pyjitpl, gc, memmgr from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler @@ -24,21 +25,17 @@ from pypy.jit.metainterp.jitdriver import JitDriverStaticData from pypy.jit.codewriter import support, codewriter from pypy.jit.codewriter.policy import JitPolicy -from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE # ____________________________________________________________ # Bootstrapping -def apply_jit(translator, backend_name="auto", debug_level=DEBUG_STEPS, - inline=False, - **kwds): +def apply_jit(translator, backend_name="auto", inline=False, **kwds): if 'CPUClass' not in kwds: from pypy.jit.backend.detect_cpu import getcpuclass kwds['CPUClass'] = getcpuclass(backend_name) - if debug_level > DEBUG_OFF: - ProfilerClass = Profiler - else: - ProfilerClass = EmptyProfiler + ProfilerClass = Profiler + # Always use Profiler here, which should have a very low impact. + # Otherwise you can try with ProfilerClass = EmptyProfiler. warmrunnerdesc = WarmRunnerDesc(translator, translate_support_code=True, listops=True, @@ -47,7 +44,6 @@ **kwds) for jd in warmrunnerdesc.jitdrivers_sd: jd.warmstate.set_param_inlining(inline) - jd.warmstate.set_param_debug(debug_level) warmrunnerdesc.finish() translator.warmrunnerdesc = warmrunnerdesc # for later debugging @@ -66,7 +62,7 @@ def jittify_and_run(interp, graph, args, repeat=1, backendopt=False, trace_limit=sys.maxint, - debug_level=DEBUG_STEPS, inline=False, **kwds): + inline=False, loop_longevity=0, **kwds): from pypy.config.config import ConfigError translator = interp.typer.annotator.translator try: @@ -83,7 +79,7 @@ jd.warmstate.set_param_trace_eagerness(2) # for tests jd.warmstate.set_param_trace_limit(trace_limit) jd.warmstate.set_param_inlining(inline) - jd.warmstate.set_param_debug(debug_level) + jd.warmstate.set_param_loop_longevity(loop_longevity) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -151,6 +147,7 @@ optimizer=None, ProfilerClass=EmptyProfiler, **kwds): pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) + self.memory_manager = memmgr.MemoryManager() self.build_cpu(CPUClass, **kwds) self.find_portals() self.codewriter = codewriter.CodeWriter(self.cpu, self.jitdrivers_sd) @@ -162,6 +159,8 @@ self.check_access_directly_sanity(graphs) if backendopt: self.prejit_optimizations(policy, graphs) + elif self.opt.listops: + self.prejit_optimizations_minimal_inline(policy, graphs) self.build_meta_interp(ProfilerClass) self.make_args_specifications() @@ -259,6 +258,10 @@ remove_asserts=True, really_remove_asserts=True) + def prejit_optimizations_minimal_inline(self, policy, graphs): + from pypy.translator.backendopt.inline import auto_inline_graphs + auto_inline_graphs(self.translator, graphs, 0.01) + def build_cpu(self, CPUClass, translate_support_code=False, no_stats=False, **kwds): assert CPUClass is not None @@ -713,7 +716,7 @@ loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) except JitException, e: return handle_jitexception(e) - fail_descr = self.cpu.execute_token(loop_token) + fail_descr = self.execute_token(loop_token) jd._assembler_call_helper = assembler_call_helper # for debugging jd._assembler_helper_ptr = self.helper_func( @@ -807,3 +810,10 @@ py.test.skip("rewrite_force_virtual: port it to ootype") all_graphs = self.translator.graphs vrefinfo.replace_force_virtual_with_call(all_graphs) + + # ____________________________________________________________ + + def execute_token(self, loop_token): + fail_descr = self.cpu.execute_token(loop_token) + self.memory_manager.keep_loop_alive(loop_token) + return fail_descr Modified: pypy/branch/jitypes2/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/warmstate.py Fri Dec 3 11:04:29 2010 @@ -1,4 +1,4 @@ -import sys +import sys, weakref from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rffi from pypy.rpython.ootypesystem import ootype from pypy.rpython.annlowlevel import hlstr, llstr, cast_base_ptr_to_instance @@ -9,7 +9,6 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.jit import (PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL, OPTIMIZER_NO_PERFECTSPEC) -from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import history @@ -150,9 +149,34 @@ # counter == -1: there is an entry bridge for this cell # counter == -2: tracing is currently going on for this cell counter = 0 - compiled_merge_points = None + compiled_merge_points_wref = None # list of weakrefs to LoopToken dont_trace_here = False - entry_loop_token = None + wref_entry_loop_token = None # (possibly) one weakref to LoopToken + + def get_compiled_merge_points(self): + result = [] + if self.compiled_merge_points_wref is not None: + for wref in self.compiled_merge_points_wref: + looptoken = wref() + if looptoken is not None: + result.append(looptoken) + return result + + def set_compiled_merge_points(self, looptokens): + self.compiled_merge_points_wref = [self._makeref(token) + for token in looptokens] + + def get_entry_loop_token(self): + if self.wref_entry_loop_token is not None: + return self.wref_entry_loop_token() + return None + + def set_entry_loop_token(self, looptoken): + self.wref_entry_loop_token = self._makeref(looptoken) + + def _makeref(self, looptoken): + assert looptoken is not None + return weakref.ref(looptoken) # ____________________________________________________________ @@ -165,6 +189,8 @@ "NOT_RPYTHON" self.warmrunnerdesc = warmrunnerdesc self.jitdriver_sd = jitdriver_sd + if warmrunnerdesc is not None: # for tests + self.cpu = warmrunnerdesc.cpu try: self.profiler = warmrunnerdesc.metainterp_sd.profiler except AttributeError: # for tests @@ -176,7 +202,7 @@ meth(default_value) def set_param_threshold(self, threshold): - if threshold < 0: + if threshold <= 0: self.increment_threshold = 0 # never reach the THRESHOLD_LIMIT return if threshold < 2: @@ -209,10 +235,11 @@ else: raise ValueError("unknown optimizer") - def set_param_debug(self, value): - self.debug_level = value - if self.profiler is not None: - self.profiler.set_printing(value >= DEBUG_PROFILE) + def set_param_loop_longevity(self, value): + # note: it's a global parameter, not a per-jitdriver one + if (self.warmrunnerdesc is not None and + self.warmrunnerdesc.memory_manager is not None): # all for tests + self.warmrunnerdesc.memory_manager.set_max_age(value) def disable_noninlinable_function(self, greenkey): cell = self.jit_cell_at_key(greenkey) @@ -225,12 +252,15 @@ def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): cell = self.jit_cell_at_key(greenkey) - cell.counter = -1 - old_token = cell.entry_loop_token - cell.entry_loop_token = entry_loop_token + old_token = cell.get_entry_loop_token() + cell.set_entry_loop_token(entry_loop_token) + cell.counter = -1 # valid entry bridge attached if old_token is not None: - cpu = self.warmrunnerdesc.cpu - cpu.redirect_call_assembler(old_token, entry_loop_token) + self.cpu.redirect_call_assembler(old_token, entry_loop_token) + # entry_loop_token is also kept alive by any loop that used + # to point to old_token. Actually freeing old_token early + # is a pointless optimization (it is tiny). + old_token.record_jump_to(entry_loop_token) # ---------- @@ -239,7 +269,8 @@ if hasattr(self, 'maybe_compile_and_run'): return self.maybe_compile_and_run - metainterp_sd = self.warmrunnerdesc.metainterp_sd + warmrunnerdesc = self.warmrunnerdesc + metainterp_sd = warmrunnerdesc.metainterp_sd jitdriver_sd = self.jitdriver_sd vinfo = jitdriver_sd.virtualizable_info index_of_virtualizable = jitdriver_sd.index_of_virtualizable @@ -297,23 +328,27 @@ assert cell.counter == -1 if not confirm_enter_jit(*args): return + loop_token = cell.get_entry_loop_token() + if loop_token is None: # it was a weakref that has been freed + cell.counter = 0 + return # machine code was already compiled for these greenargs # get the assembler and fill in the boxes set_future_values(*args[num_green_args:]) - loop_token = cell.entry_loop_token # ---------- execute assembler ---------- while True: # until interrupted by an exception metainterp_sd.profiler.start_running() debug_start("jit-running") - fail_descr = metainterp_sd.cpu.execute_token(loop_token) + fail_descr = warmrunnerdesc.execute_token(loop_token) debug_stop("jit-running") metainterp_sd.profiler.end_running() + loop_token = None # for test_memmgr if vinfo is not None: vinfo.reset_vable_token(virtualizable) loop_token = fail_descr.handle_fail(metainterp_sd, jitdriver_sd) - + maybe_compile_and_run._dont_inline_ = True self.maybe_compile_and_run = maybe_compile_and_run return maybe_compile_and_run @@ -459,7 +494,7 @@ warmrunnerdesc = self.warmrunnerdesc jitdriver_sd = self.jitdriver_sd - cpu = warmrunnerdesc.cpu + cpu = self.cpu vinfo = jitdriver_sd.virtualizable_info red_args_types = unrolling_iterable(jitdriver_sd._red_args_types) # @@ -508,10 +543,11 @@ if hasattr(self, 'get_location_str'): return # + warmrunnerdesc = self.warmrunnerdesc unwrap_greenkey = self.make_unwrap_greenkey() jit_getter = self.make_jitcell_getter() jd = self.jitdriver_sd - cpu = self.warmrunnerdesc.cpu + cpu = self.cpu def can_inline_greenargs(*greenargs): if can_never_inline(*greenargs): @@ -529,11 +565,16 @@ def get_assembler_token(greenkey, redboxes): # 'redboxes' is only used to know the types of red arguments cell = self.jit_cell_at_key(greenkey) - if cell.entry_loop_token is None: + entry_loop_token = cell.get_entry_loop_token() + if entry_loop_token is None: from pypy.jit.metainterp.compile import compile_tmp_callback - cell.entry_loop_token = compile_tmp_callback(cpu, jd, greenkey, - redboxes) - return cell.entry_loop_token + if cell.counter == -1: # used to be a valid entry bridge, + cell.counter = 0 # but was freed in the meantime. + memmgr = warmrunnerdesc.memory_manager + entry_loop_token = compile_tmp_callback(cpu, jd, greenkey, + redboxes, memmgr) + cell.set_entry_loop_token(entry_loop_token) + return entry_loop_token self.get_assembler_token = get_assembler_token # Modified: pypy/branch/jitypes2/pypy/jit/tl/spli/interpreter.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/tl/spli/interpreter.py (original) +++ pypy/branch/jitypes2/pypy/jit/tl/spli/interpreter.py Fri Dec 3 11:04:29 2010 @@ -105,26 +105,32 @@ self.stack_depth += 1 def pop(self): - self.stack_depth -= 1 - val = self.value_stack[self.stack_depth] - self.value_stack[self.stack_depth] = None + sd = self.stack_depth - 1 + assert sd >= 0 + self.stack_depth = sd + val = self.value_stack[sd] + self.value_stack[sd] = None return val def pop_many(self, n): return [self.pop() for i in range(n)] def peek(self): - return self.value_stack[self.stack_depth - 1] + sd = self.stack_depth - 1 + assert sd >= 0 + return self.value_stack[sd] def POP_TOP(self, _, next_instr, code): self.pop() return next_instr def LOAD_FAST(self, name_index, next_instr, code): + assert name_index >= 0 self.push(self.locals[name_index]) return next_instr def STORE_FAST(self, name_index, next_instr, code): + assert name_index >= 0 self.locals[name_index] = self.pop() return next_instr Modified: pypy/branch/jitypes2/pypy/jit/tl/tl.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/tl/tl.py (original) +++ pypy/branch/jitypes2/pypy/jit/tl/tl.py Fri Dec 3 11:04:29 2010 @@ -16,32 +16,40 @@ def __init__(self, size): self = hint(self, access_directly=True, fresh_virtualizable=True) self.stack = [0] * size - self.stackpos = 0 + self.stackpos = 0 # always store a known-nonneg integer here def append(self, elem): self.stack[self.stackpos] = elem self.stackpos += 1 def pop(self): - self.stackpos -= 1 - if self.stackpos < 0: + stackpos = self.stackpos - 1 + if stackpos < 0: raise IndexError - return self.stack[self.stackpos] + self.stackpos = stackpos # always store a known-nonneg integer here + return self.stack[stackpos] def pick(self, i): - self.append(self.stack[self.stackpos - i - 1]) + n = self.stackpos - i - 1 + assert n >= 0 + self.append(self.stack[n]) def put(self, i): elem = self.pop() - self.stack[self.stackpos - i - 1] = elem + n = self.stackpos - i - 1 + assert n >= 0 + self.stack[n] = elem def roll(self, r): if r < -1: i = self.stackpos + r if i < 0: raise IndexError - elem = self.stack[self.stackpos - 1] + n = self.stackpos - 1 + assert n >= 0 + elem = self.stack[n] for j in range(self.stackpos - 2, i - 1, -1): + assert j >= 0 self.stack[j + 1] = self.stack[j] self.stack[i] = elem elif r > 1: @@ -51,7 +59,9 @@ elem = self.stack[i] for j in range(i, self.stackpos - 1): self.stack[j] = self.stack[j + 1] - self.stack[self.stackpos - 1] = elem + n = self.stackpos - 1 + assert n >= 0 + self.stack[n] = elem def make_interp(supports_call): Modified: pypy/branch/jitypes2/pypy/jit/tool/jitoutput.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/tool/jitoutput.py (original) +++ pypy/branch/jitypes2/pypy/jit/tool/jitoutput.py Fri Dec 3 11:04:29 2010 @@ -27,6 +27,10 @@ (('nvirtuals',), '^nvirtuals:\s+(\d+)$'), (('nvholes',), '^nvholes:\s+(\d+)$'), (('nvreused',), '^nvreused:\s+(\d+)$'), + (('total_compiled_loops',), '^Total # of loops:\s+(\d+)$'), + (('total_compiled_bridges',), '^Total # of bridges:\s+(\d+)$'), + (('total_freed_loops',), '^Freed # of loops:\s+(\d+)$'), + (('total_freed_bridges',), '^Freed # of bridges:\s+(\d+)$'), ] class Ops(object): Modified: pypy/branch/jitypes2/pypy/jit/tool/log-template.gnumeric ============================================================================== Binary files. No diff available. Modified: pypy/branch/jitypes2/pypy/jit/tool/log2gnumeric.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/tool/log2gnumeric.py (original) +++ pypy/branch/jitypes2/pypy/jit/tool/log2gnumeric.py Fri Dec 3 11:04:29 2010 @@ -14,42 +14,85 @@ import re, sys import gzip +import optparse -def main(): + +def main(logname, options): logname = sys.argv[1] outname = logname + '.gnumeric' data = open(logname).read() data = data.replace('\n', '') - xml = gzip.open('log-template.gnumeric').read() + minclock, maxclock = get_clock_range(data) + time0 = minclock # we want "relative clocks" + maxtime = maxclock-time0 # - xml = replace_sheet(xml, 'translation-task', tasks_rows(data)) - xml = replace_sheet(xml, 'gc-collect', gc_collect_rows(data)) + xml = gzip.open('log-template.gnumeric').read() + xml = replace_sheet(xml, 'translation-task', tasks_rows(time0, data)) + xml = replace_sheet(xml, 'gc-collect', gc_collect_rows(time0, data)) + xml = replace_sheet(xml, 'loops', loops_rows(time0, data)) + xml = replace_sheet(xml, 'vmrss', vmrss_rows(logname + '.vmrss', maxtime)) + xml = replace_sheet(xml, 'cpython-vmrss', vmrss_rows(options.cpython_vmrss, maxtime)) # out = gzip.open(outname, 'wb') out.write(xml) out.close() +# ======================================================================== +# functions to manipulate gnumeric files +# ======================================================================== + def replace_sheet(xml, sheet_name, data): - pattern = '(%s.*?).*?' + pattern = '%s.*?(.*?)' regex = re.compile(pattern % sheet_name, re.DOTALL) cells = gen_cells(data) - xml2 = regex.sub(r'\1%s' % cells, xml) - assert xml != xml2 + match = regex.search(xml) + if not match: + print 'Cannot find sheet %s' % sheet_name + return xml + a, b = match.span(1) + xml2 = xml[:a] + cells + xml[b:] return xml2 def gen_cells(data): + # values for the ValueType attribute + ValueType_Empty = 'ValueType="10"' + ValueType_Number = 'ValueType="40"' + ValueType_String = 'ValueType="60"' + # parts = [] - parts.append(' ') + parts.append('') for i, row in enumerate(data): for j, val in enumerate(row): - cell = ' %s' - parts.append(cell % (i, j, val)) + if val is None: + attr = ValueType_Empty + val = '' + elif isinstance(val, (int, long, float)): + attr = ValueType_Number + else: + attr = ValueType_String + cell = ' %s' + parts.append(cell % (i, j, attr, val)) parts.append(' ') - return ''.join(parts) + return '\n'.join(parts) -def gc_collect_rows(data, show_header=True): +# ======================================================================== +# functions to extract various data from the logs +# ======================================================================== + +CLOCK_FACTOR = 1 +def read_clock(x): + timestamp = int(x, 16) + return timestamp / CLOCK_FACTOR + +def get_clock_range(data): + s = r"\[([0-9a-f]+)\] " + r = re.compile(s) + clocks = [read_clock(x) for x in r.findall(data)] + return min(clocks), max(clocks) + +def gc_collect_rows(time0, data): s = r""" ----------- Full collection ------------------ \| used before collection: @@ -63,19 +106,75 @@ \[([0-9a-f]+)\] gc-collect\}""" # r = re.compile(s.replace('\n', '')) + yield 'clock', 'gc-before', 'gc-after' for a,b,c,d,e,f in r.findall(data): - yield int(f, 16), int(a)+int(b), int(c)+int(d) + clock = read_clock(f) - time0 + yield clock, int(a)+int(b), int(c)+int(d) -def tasks_rows(data): +def tasks_rows(time0, data): s = r""" -\{translation-task +\[([0-9a-f]+)\] \{translation-task starting ([\w-]+) -\[([0-9a-f]+)\] translation-task\}""" +""" # r = re.compile(s.replace('\n', '')) + yield 'clock', None, 'task' for a,b in r.findall(data): - yield int(b, 16), 1, a + clock = read_clock(a) - time0 + yield clock, 1, b + + +def loops_rows(time0, data): + s = r""" +\[([0-9a-f]+)\] \{jit-mem-looptoken-(alloc|free) +(.*?)\[ +""" + # + r = re.compile(s.replace('\n', '')) + yield 'clock', 'total', 'loops', 'bridges' + loops = 0 + bridges = 0 + fake_total = 0 + for clock, action, text in r.findall(data): + clock = read_clock(clock) - time0 + if text.startswith('allocating Loop #'): + loops += 1 + elif text.startswith('allocating Bridge #'): + bridges += 1 + elif text.startswith('freeing Loop #'): + match = re.match('freeing Loop # .* with ([0-9]*) attached bridges', text) + loops -=1 + bridges -= int(match.group(1)) + total = loops+bridges + yield clock, loops+bridges, loops, bridges + + +def vmrss_rows(filename, maxtime): + lines = [] + if options.cpython_vmrss: + try: + lines = open(filename).readlines() + except IOError: + print 'Warning: cannot find file %s, skipping this sheet' + for row in vmrss_rows_impl(lines, maxtime): + yield row + +def vmrss_rows_impl(lines, maxtime): + yield 'inferred clock', 'VmRSS' + numlines = len(lines) + for i, line in enumerate(lines): + mem = int(line) + clock = maxtime * i // (numlines-1) + yield clock, mem if __name__ == '__main__': - main() + CLOCK_FACTOR = 1000000000.0 # report GigaTicks instead of Ticks + parser = optparse.OptionParser(usage="%prog logfile [options]") + parser.add_option('-c', '--cpython-vmrss', dest='cpython_vmrss', default=None, metavar='FILE', type=str, + help='the .vmrss file produced by CPython') + options, args = parser.parse_args() + if len(args) != 1: + parser.print_help() + sys.exit(2) + main(args[0], options) Modified: pypy/branch/jitypes2/pypy/jit/tool/test/test_jitoutput.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/tool/test/test_jitoutput.py (original) +++ pypy/branch/jitypes2/pypy/jit/tool/test/test_jitoutput.py Fri Dec 3 11:04:29 2010 @@ -1,10 +1,11 @@ import py from pypy.jit.metainterp.warmspot import ll_meta_interp -from pypy.rlib.jit import JitDriver, DEBUG_PROFILE +from pypy.rlib.jit import JitDriver from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.jitprof import Profiler, JITPROF_LINES from pypy.jit.tool.jitoutput import parse_prof +from pypy.tool.logparser import parse_log, extract_category def test_really_run(): """ This test checks whether output of jitprof did not change. @@ -21,13 +22,15 @@ cap = py.io.StdCaptureFD() try: ll_meta_interp(f, [10], CPUClass=runner.LLtypeCPU, type_system='lltype', - ProfilerClass=Profiler, debug_level=DEBUG_PROFILE) + ProfilerClass=Profiler) finally: out, err = cap.reset() - err = "\n".join(err.splitlines()[-JITPROF_LINES:]) - print err - assert err.count("\n") == JITPROF_LINES - 1 - info = parse_prof(err) + + log = parse_log(err.splitlines(True)) + err_sections = list(extract_category(log, 'jit-summary')) + [err1] = err_sections # there should be exactly one jit-summary + assert err1.count("\n") == JITPROF_LINES + info = parse_prof(err1) # assert did not crash # asserts below are a bit delicate, possibly they might be deleted assert info.tracing_no == 1 @@ -60,6 +63,10 @@ nvirtuals: 13 nvholes: 14 nvreused: 15 +Total # of loops: 100 +Total # of bridges: 300 +Freed # of loops: 99 +Freed # of bridges: 299 ''' def test_parse(): Modified: pypy/branch/jitypes2/pypy/module/__pypy__/__init__.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/__pypy__/__init__.py (original) +++ pypy/branch/jitypes2/pypy/module/__pypy__/__init__.py Fri Dec 3 11:04:29 2010 @@ -11,6 +11,11 @@ 'internal_repr' : 'interp_magic.internal_repr', 'bytebuffer' : 'bytebuffer.bytebuffer', 'identity_dict' : 'interp_identitydict.W_IdentityDict', + 'debug_start' : 'interp_debug.debug_start', + 'debug_print' : 'interp_debug.debug_print', + 'debug_stop' : 'interp_debug.debug_stop', + 'debug_print_once' : 'interp_debug.debug_print_once', + 'builtinify' : 'interp_magic.builtinify', } def setup_after_space_initialization(self): Modified: pypy/branch/jitypes2/pypy/module/__pypy__/interp_magic.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/__pypy__/interp_magic.py (original) +++ pypy/branch/jitypes2/pypy/module/__pypy__/interp_magic.py Fri Dec 3 11:04:29 2010 @@ -51,3 +51,9 @@ return space.newtuple([space.newint(cache.hits.get(name, 0)), space.newint(cache.misses.get(name, 0))]) mapdict_cache_counter.unwrap_spec = [ObjSpace, str] + +def builtinify(space, w_func): + from pypy.interpreter.function import Function, BuiltinFunction + func = space.interp_w(Function, w_func) + bltn = BuiltinFunction(func) + return space.wrap(bltn) Modified: pypy/branch/jitypes2/pypy/module/__pypy__/test/test_special.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/__pypy__/test/test_special.py (original) +++ pypy/branch/jitypes2/pypy/module/__pypy__/test/test_special.py Fri Dec 3 11:04:29 2010 @@ -21,3 +21,18 @@ def test_cpumodel(self): import __pypy__ assert hasattr(__pypy__, 'cpumodel') + + def test_builtinify(self): + import __pypy__ + class A(object): + a = lambda *args: args + b = __pypy__.builtinify(a) + my = A() + assert my.a() == (my,) + assert my.b() == () + assert A.a(my) == (my,) + assert A.b(my) == (my,) + assert A.a.im_func(my) == (my,) + assert not hasattr(A.b, 'im_func') + assert A.a is not A.__dict__['a'] + assert A.b is A.__dict__['b'] Modified: pypy/branch/jitypes2/pypy/module/_minimal_curses/__init__.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_minimal_curses/__init__.py (original) +++ pypy/branch/jitypes2/pypy/module/_minimal_curses/__init__.py Fri Dec 3 11:04:29 2010 @@ -4,7 +4,7 @@ try: import _minimal_curses as _curses # when running on top of pypy-c except ImportError: - import py; py.test.skip("no _curses module") # no _curses at all + raise ImportError("no _curses or _minimal_curses module") # no _curses at all from pypy.interpreter.mixedmodule import MixedModule from pypy.module._minimal_curses import fficurses Modified: pypy/branch/jitypes2/pypy/module/_pickle_support/maker.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_pickle_support/maker.py (original) +++ pypy/branch/jitypes2/pypy/module/_pickle_support/maker.py Fri Dec 3 11:04:29 2010 @@ -67,11 +67,12 @@ return space.wrap(tb) traceback_new.unwrap_spec = [ObjSpace] -def generator_new(space, frame, running): +def generator_new(space, w_frame, running): + frame = space.interp_w(PyFrame, w_frame, can_be_None=True) new_generator = GeneratorIterator(frame) new_generator.running = running return space.wrap(new_generator) -generator_new.unwrap_spec = [ObjSpace, PyFrame, int] +generator_new.unwrap_spec = [ObjSpace, W_Root, int] def xrangeiter_new(space, current, remaining, step): from pypy.module.__builtin__.functional import W_XRangeIterator Modified: pypy/branch/jitypes2/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/api.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/api.py Fri Dec 3 11:04:29 2010 @@ -321,6 +321,8 @@ 'PyCObject_Type', 'init_pycobject', 'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer', + + 'PyStructSequence_InitType', 'PyStructSequence_New', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur @@ -367,7 +369,7 @@ }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) - for cpyname in 'Method List Int Long Dict Tuple Class'.split(): + for cpyname in 'Method List Int Long Dict Class'.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } ' 'Py%sObject' % (cpyname, )) build_exported_objects() @@ -845,6 +847,7 @@ source_dir / "bufferobject.c", source_dir / "object.c", source_dir / "cobject.c", + source_dir / "structseq.c", ], separate_module_sources = [code, struct_source], export_symbols=export_symbols_eci, Modified: pypy/branch/jitypes2/pypy/module/cpyext/cdatetime.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/cdatetime.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/cdatetime.py Fri Dec 3 11:04:29 2010 @@ -4,8 +4,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, cpython_struct, PyObjectFields) from pypy.module.cpyext.import_ import PyImport_Import -from pypy.module.cpyext.typeobject import PyTypeObjectPtr, render_immortal -from pypy.module.cpyext.state import State +from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_renamer @@ -25,48 +24,26 @@ datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw', track_allocation=False) - if not we_are_translated(): - datetimeAPI_dealloc(space) - space.fromcache(State).datetimeAPI = datetimeAPI - w_datetime = PyImport_Import(space, space.wrap("datetime")) w_type = space.getattr(w_datetime, space.wrap("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DateType, w_type) w_type = space.getattr(w_datetime, space.wrap("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DateTimeType, w_type) w_type = space.getattr(w_datetime, space.wrap("time")) datetimeAPI.c_TimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_TimeType, w_type) w_type = space.getattr(w_datetime, space.wrap("timedelta")) datetimeAPI.c_DeltaType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DeltaType, w_type) return datetimeAPI -def datetimeAPI_dealloc(space): - "Used in tests only, to please the refcount checker" - if we_are_translated(): - return - datetimeAPI = space.fromcache(State).datetimeAPI - if datetimeAPI is None: - return - space.fromcache(State).datetimeAPI = None - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DateType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DateTimeType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_TimeType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DeltaType)) - lltype.free(datetimeAPI, flavor='raw') - PyDateTime_Date = PyObject PyDateTime_Time = PyObject PyDateTime_DateTime = PyObject Modified: pypy/branch/jitypes2/pypy/module/cpyext/include/tupleobject.h ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/include/tupleobject.h (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/include/tupleobject.h Fri Dec 3 11:04:29 2010 @@ -10,9 +10,19 @@ /* defined in varargswrapper.c */ PyObject * PyTuple_Pack(Py_ssize_t, ...); -#define PyTuple_SET_ITEM PyTuple_SetItem -#define PyTuple_GET_ITEM PyTuple_GetItem +typedef struct { + PyObject_HEAD + PyObject **items; + Py_ssize_t size; +} PyTupleObject; +#define PyTuple_GET_ITEM PyTuple_GetItem + +/* Macro, trading safety for speed */ +#define PyTuple_GET_SIZE(op) (((PyTupleObject *)(op))->size) + +/* Macro, *only* to be used to fill in brand new tuples */ +#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->items[i] = v) #ifdef __cplusplus } Modified: pypy/branch/jitypes2/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/sequence.py Fri Dec 3 11:04:29 2010 @@ -116,6 +116,14 @@ This is the equivalent of the Python expression o1 + o2.""" return space.add(w_o1, w_o2) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PySequence_Contains(space, w_obj, w_value): + """Determine if o contains value. If an item in o is equal to value, + return 1, otherwise return 0. On error, return -1. This is + equivalent to the Python expression value in o.""" + w_res = space.contains(w_obj, w_value) + return space.int_w(w_res) + @cpython_api([PyObject], PyObject) def PySeqIter_New(space, w_seq): """Return an iterator that works with a general sequence object, seq. The Modified: pypy/branch/jitypes2/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/slotdefs.py Fri Dec 3 11:04:29 2010 @@ -6,7 +6,7 @@ unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, setattrofunc, lenfunc, ssizeargfunc, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, hashfunc, - descrgetfunc, descrsetfunc) + descrgetfunc, descrsetfunc, objobjproc) from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -28,14 +28,14 @@ def check_num_args(space, ob, n): from pypy.module.cpyext.tupleobject import PyTuple_CheckExact, \ - PyTuple_GET_SIZE + _PyTuple_Size_Fast if not PyTuple_CheckExact(space, ob): raise OperationError(space.w_SystemError, space.wrap("PyArg_UnpackTuple() argument list is not a tuple")) - if n == PyTuple_GET_SIZE(space, ob): + if n == _PyTuple_Size_Fast(space, ob): return raise operationerrfmt(space.w_TypeError, - "expected %d arguments, got %d", n, PyTuple_GET_SIZE(space, ob)) + "expected %d arguments, got %d", n, _PyTuple_Size_Fast(space, ob)) def wrap_init(space, w_self, w_args, func, w_kwargs): func_init = rffi.cast(initproc, func) @@ -156,6 +156,15 @@ if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) +def wrap_objobjproc(space, w_self, w_args, func): + func_target = rffi.cast(objobjproc, func) + check_num_args(space, w_args, 1) + w_value, = space.fixedview(w_args) + res = generic_cpy_call(space, func_target, w_self, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.wrap(res) + def wrap_ssizessizeargfunc(space, w_self, w_args, func): func_target = rffi.cast(ssizessizeargfunc, func) check_num_args(space, w_args, 2) Modified: pypy/branch/jitypes2/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/state.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/state.py Fri Dec 3 11:04:29 2010 @@ -5,8 +5,6 @@ class State: - datetimeAPI = None # used in tests - def __init__(self, space): self.space = space self.reset() Modified: pypy/branch/jitypes2/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/stubs.py Fri Dec 3 11:04:29 2010 @@ -2289,13 +2289,6 @@ in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PySequence_Contains(space, o, value): - """Determine if o contains value. If an item in o is equal to value, - return 1, otherwise return 0. On error, return -1. This is - equivalent to the Python expression value in o.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PySequence_Index(space, o, value): """Return the first index i for which o[i] == value. On error, return Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_arraymodule.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_arraymodule.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_arraymodule.py Fri Dec 3 11:04:29 2010 @@ -14,6 +14,7 @@ arr.append(4) assert arr.tolist() == [1, 2, 3, 4] assert len(arr) == 4 + self.cleanup_references() def test_iter(self): module = self.import_module(name='array') @@ -22,6 +23,7 @@ for i in arr: sum += i assert sum == 6 + self.cleanup_references() def test_index(self): module = self.import_module(name='array') @@ -32,6 +34,7 @@ assert arr.tolist() == [1,2,4] arr[2] = 99 assert arr.tolist() == [1,2,99] + self.cleanup_references() def test_slice_get(self): module = self.import_module(name='array') @@ -40,3 +43,4 @@ assert arr[1:].tolist() == [2,3,4] assert arr[:2].tolist() == [1,2] assert arr[1:3].tolist() == [2,3] + self.cleanup_references() Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_cpyext.py Fri Dec 3 11:04:29 2010 @@ -1,10 +1,12 @@ import sys +import weakref import os.path import py from pypy.conftest import gettestobjspace from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import interp2app from pypy.rpython.lltypesystem import rffi, lltype, ll2ctypes from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator import platform @@ -79,6 +81,23 @@ self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) class LeakCheckingTest(object): + @staticmethod + def cleanup_references(space): + state = space.fromcache(RefcountState) + + import gc; gc.collect() + # Clear all lifelines, objects won't resurrect + for w_obj, obj in state.lifeline_dict._dict.items(): + if w_obj not in state.py_objects_w2r: + state.lifeline_dict.set(w_obj, None) + del obj + import gc; gc.collect() + + for w_obj in state.non_heaptypes_w: + Py_DecRef(space, w_obj) + state.non_heaptypes_w[:] = [] + state.reset_borrowed_references() + def check_and_print_leaks(self): # check for sane refcnts import gc @@ -89,13 +108,6 @@ lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) - # Clear all lifelines, objects won't resurrect - for w_obj, obj in state.lifeline_dict._dict.items(): - if w_obj not in state.py_objects_w2r: - state.lifeline_dict.set(w_obj, None) - del obj - gc.collect() - for w_obj, obj in state.py_objects_w2r.iteritems(): base_refcnt = self.frozen_refcounts.get(w_obj) delta = obj.c_ob_refcnt @@ -105,7 +117,12 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = state.lifeline_dict.get(w_obj) + try: + weakref.ref(w_obj) + except TypeError: + lifeline = None + else: + lifeline = state.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: @@ -137,6 +154,8 @@ state = cls.space.fromcache(RefcountState) state.non_heaptypes_w[:] = [] + cls.w_cleanup_references = cls.space.wrap(interp2app(cls.cleanup_references)) + def compile_module(self, name, **kwds): """ Build an extension module linked against the cpyext api library. @@ -256,13 +275,7 @@ def teardown_method(self, func): for name in self.imported_module_names: self.unimport_module(name) - state = self.space.fromcache(RefcountState) - for w_obj in state.non_heaptypes_w: - Py_DecRef(self.space, w_obj) - state.non_heaptypes_w[:] = [] - state.reset_borrowed_references() - from pypy.module.cpyext import cdatetime - cdatetime.datetimeAPI_dealloc(self.space) + self.cleanup_references(self.space) if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." @@ -532,16 +545,17 @@ PyObject *true = Py_True; PyObject *tup = NULL; int refcnt = true->ob_refcnt; - int refcnt_after; + int refcnt_middle, refcnt_after; tup = PyTuple_New(1); Py_INCREF(true); if (PyTuple_SetItem(tup, 0, true) < 0) return NULL; - refcnt_after = true->ob_refcnt; + refcnt_middle = true->ob_refcnt; Py_DECREF(tup); - fprintf(stderr, "REFCNT2 %i %i\\n", refcnt, refcnt_after); - return PyBool_FromLong(refcnt_after == refcnt); + refcnt_after = true->ob_refcnt; + fprintf(stderr, "REFCNT2 %i %i %i\\n", refcnt, refcnt_middle, refcnt_after); + return PyBool_FromLong(refcnt_after == refcnt && refcnt_middle == refcnt+1); } static PyMethodDef methods[] = { Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_datetime.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_datetime.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_datetime.py Fri Dec 3 11:04:29 2010 @@ -92,9 +92,20 @@ PyDateTimeAPI->TimeType, PyDateTimeAPI->DeltaType); """), + ("clear_types", "METH_NOARGS", + """ + Py_DECREF(PyDateTimeAPI->DateType); + Py_DECREF(PyDateTimeAPI->DateTimeType); + Py_DECREF(PyDateTimeAPI->TimeType); + Py_DECREF(PyDateTimeAPI->DeltaType); + Py_RETURN_NONE; + """ + ) ]) import datetime assert module.get_types() == (datetime.date, datetime.datetime, datetime.time, datetime.timedelta) + module.clear_types() + self.cleanup_references() Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_sequence.py Fri Dec 3 11:04:29 2010 @@ -70,3 +70,11 @@ assert space.unwrap(space.next(w_iter)) == 2 exc = raises(OperationError, space.next, w_iter) assert exc.value.match(space, space.w_StopIteration) + + def test_contains(self, space, api): + w_t = space.wrap((1, 'ha')) + assert api.PySequence_Contains(w_t, space.wrap(u'ha')) + assert not api.PySequence_Contains(w_t, space.wrap(2)) + assert api.PySequence_Contains(space.w_None, space.wrap(2)) == -1 + assert api.PyErr_Occurred() + api.PyErr_Clear() Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_tupleobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_tupleobject.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_tupleobject.py Fri Dec 3 11:04:29 2010 @@ -7,24 +7,34 @@ class TestTupleObject(BaseApiTest): def test_tupleobject(self, space, api): assert not api.PyTuple_Check(space.w_None) - assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 + #assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 XXX atuple = space.newtuple([0, 1, 'yay']) assert api.PyTuple_Size(atuple) == 3 - assert api.PyTuple_GET_SIZE(atuple) == 3 - raises(TypeError, api.PyTuple_Size(space.newlist([]))) + #raises(TypeError, api.PyTuple_Size(space.newlist([]))) XXX api.PyErr_Clear() def test_tuple_resize(self, space, api): - py_tuple = api.PyTuple_New(3) + ref_tup = api.PyTuple_New(3) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - ar[0] = rffi.cast(PyObject, make_ref(space, py_tuple)) + ar[0] = rffi.cast(PyObject, ref_tup) api._PyTuple_Resize(ar, 2) - py_tuple = from_ref(space, ar[0]) - assert len(py_tuple.wrappeditems) == 2 + assert ar[0] == rffi.cast(PyObject, ref_tup) + # ^^^ our _PyTuple_Resize does not actually need to change the ptr so far + assert api.PyTuple_Size(ar[0]) == 2 api._PyTuple_Resize(ar, 10) - py_tuple = from_ref(space, ar[0]) - assert len(py_tuple.wrappeditems) == 10 + assert api.PyTuple_Size(ar[0]) == 10 api.Py_DecRef(ar[0]) lltype.free(ar, flavor='raw') + + def test_tuple_setup(self, space, api): + ref_tup = api.PyTuple_New(2) + ref0 = make_ref(space, space.wrap(123)) + api.PyTuple_SetItem(ref_tup, 0, ref0) + ref1 = make_ref(space, space.wrap(456)) + api.PyTuple_SetItem(ref_tup, 1, ref1) + + w_tup = from_ref(space, ref_tup) + assert space.is_true(space.eq(w_tup, space.wrap((123, 456)))) + api.Py_DecRef(ref_tup) Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_typeobject.py Fri Dec 3 11:04:29 2010 @@ -20,6 +20,7 @@ assert type(obj) is module.fooType print "type of obj has type", type(type(obj)) print "type of type of obj has type", type(type(type(obj))) + self.cleanup_references() def test_typeobject_method_descriptor(self): module = self.import_module(name='foo') @@ -38,6 +39,7 @@ print obj.foo assert obj.foo == 42 assert obj.int_member == obj.foo + self.cleanup_references() def test_typeobject_data_member(self): module = self.import_module(name='foo') @@ -54,6 +56,7 @@ raises(SystemError, "obj.broken_member = 42") assert module.fooType.broken_member.__doc__ is None assert module.fooType.object_member.__doc__ == "A Python object." + self.cleanup_references() def test_typeobject_object_member(self): module = self.import_module(name='foo') @@ -74,6 +77,7 @@ obj.set_foo = 32 assert obj.foo == 32 + self.cleanup_references() def test_typeobject_string_member(self): module = self.import_module(name='foo') @@ -91,6 +95,7 @@ assert obj.char_member == "a" raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") + self.cleanup_references() def test_staticmethod(self): module = self.import_module(name="foo") @@ -98,6 +103,7 @@ assert obj.foo == 42 obj2 = obj.create() assert obj2.foo == 42 + self.cleanup_references() def test_new(self): module = self.import_module(name='foo') @@ -118,7 +124,8 @@ return self assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) - + self.cleanup_references() + def test_init(self): module = self.import_module(name="foo") newobj = module.FuuType() @@ -137,6 +144,7 @@ newobj = Fuu2() assert newobj.get_val() == 42 assert newobj.foobar == 32 + self.cleanup_references() def test_metatype(self): module = self.import_module(name='foo') @@ -145,6 +153,7 @@ assert isinstance(x, type) assert isinstance(x, module.MetaType) x() + self.cleanup_references() def test_metaclass_compatible(self): # metaclasses should not conflict here @@ -153,7 +162,9 @@ assert type(module.fooType).__mro__ == (type, object) y = module.MetaType('other', (module.fooType,), {}) assert isinstance(y, module.MetaType) - y() + x = y() + del x, y + self.cleanup_references() def test_sre(self): module = self.import_module(name='_sre') @@ -172,17 +183,21 @@ assert "groupdict" in dir(m) re._cache.clear() re._cache_repl.clear() + del prog, m + self.cleanup_references() def test_init_error(self): module = self.import_module("foo") raises(ValueError, module.InitErrType) - + self.cleanup_references() + def test_cmps(self): module = self.import_module("comparisons") cmpr = module.CmpType() assert cmpr == 3 assert cmpr != 42 - + self.cleanup_references() + def test_hash(self): module = self.import_module("comparisons") cmpr = module.CmpType() @@ -191,6 +206,7 @@ d[cmpr] = 72 assert d[cmpr] == 72 assert d[3] == 72 + self.cleanup_references() def test_descriptor(self): module = self.import_module("foo") @@ -205,6 +221,7 @@ assert obj.y == (prop, 2) del obj.x assert obj.z == prop + self.cleanup_references() def test_tp_dict(self): foo = self.import_module("foo") @@ -226,6 +243,7 @@ ]) obj = foo.new() assert module.read_tp_dict(obj) == foo.fooType.copy + self.cleanup_references() class TestTypes(BaseApiTest): Modified: pypy/branch/jitypes2/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/tupleobject.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/tupleobject.py Fri Dec 3 11:04:29 2010 @@ -1,55 +1,144 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers) + build_type_checkers, PyObjectFields, + cpython_struct, bootstrap_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - borrow_from, make_ref, from_ref) + borrow_from, make_ref, from_ref, make_typedescr, get_typedescr, Reference, + track_reference) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall -from pypy.objspace.std.tupleobject import W_TupleObject +## +## Implementation of PyTupleObject +## =============================== +## +## We have the same problem as PyStringObject: a PyTupleObject can be +## initially used in a read-write way with PyTuple_New(), PyTuple_SetItem() +## and _PyTuple_Resize(). +## +## The 'size' and 'items' fields of a PyTupleObject are always valid. +## Apart from that detail, see the big comment in stringobject.py for +## more information. +## + +ARRAY_OF_PYOBJ = rffi.CArrayPtr(PyObject) +PyTupleObjectStruct = lltype.ForwardReference() +PyTupleObject = lltype.Ptr(PyTupleObjectStruct) +PyTupleObjectFields = PyObjectFields + \ + (("items", ARRAY_OF_PYOBJ), ("size", Py_ssize_t)) +cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) + + at bootstrap_function +def init_tupleobject(space): + "Type description of PyTupleObject" + make_typedescr(space.w_tuple.instancetypedef, + basestruct=PyTupleObject.TO, + attach=tuple_attach, + dealloc=tuple_dealloc, + realize=tuple_realize) PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple") +def new_empty_tuple(space, length): + """ + Allocate a PyTupleObject and its array, but without a corresponding + interpreter object. The array items may be mutated, until + tuple_realize() is called. + """ + typedescr = get_typedescr(space.w_tuple.instancetypedef) + py_obj = typedescr.allocate(space, space.w_tuple) + py_tup = rffi.cast(PyTupleObject, py_obj) + + py_tup.c_items = lltype.malloc(ARRAY_OF_PYOBJ.TO, length, + flavor='raw', zero=True) + py_tup.c_size = length + return py_tup + +def tuple_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PyTupleObject with the given tuple object. + """ + items_w = space.fixedview(w_obj) + py_tup = rffi.cast(PyTupleObject, py_obj) + py_tup.c_items = lltype.nullptr(ARRAY_OF_PYOBJ.TO) + py_tup.c_size = len(items_w) + +def tuple_realize(space, py_obj): + """ + Creates the tuple in the interpreter. The PyTupleObject items array + must not be modified after this call. + """ + py_tup = rffi.cast(PyTupleObject, py_obj) + # If your CPython extension creates a self-referential tuple + # with PyTuple_SetItem(), you loose. + c_items = py_tup.c_items + items_w = [from_ref(space, c_items[i]) for i in range(py_tup.c_size)] + w_obj = space.newtuple(items_w) + track_reference(space, py_obj, w_obj) + return w_obj + + at cpython_api([PyObject], lltype.Void, external=False) +def tuple_dealloc(space, py_obj): + """Frees allocated PyTupleObject resources. + """ + py_tup = rffi.cast(PyTupleObject, py_obj) + if py_tup.c_items: + for i in range(py_tup.c_size): + Py_DecRef(space, py_tup.c_items[i]) + lltype.free(py_tup.c_items, flavor="raw") + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) + +#_______________________________________________________________________ + @cpython_api([Py_ssize_t], PyObject) def PyTuple_New(space, size): - return space.newtuple([space.w_None] * size) + return rffi.cast(PyObject, new_empty_tuple(space, size)) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PyTuple_SetItem(space, w_t, pos, w_obj): - if not PyTuple_Check(space, w_t): - # XXX this should also steal a reference, test it!!! - PyErr_BadInternalCall(space) - assert isinstance(w_t, W_TupleObject) - w_t.wrappeditems[pos] = w_obj - Py_DecRef(space, w_obj) # SetItem steals a reference! +def PyTuple_SetItem(space, ref, pos, ref_item): + # XXX steals a reference at the level of PyObjects. Don't try to + # XXX call this function with an interpreter object as ref_item! + + # XXX do PyTuple_Check, without forcing ref as an interpreter object + # XXX -- then if it fails it should also steal a reference, test it!!! + ref_tup = rffi.cast(PyTupleObject, ref) + if not ref_tup.c_items: + msg = "PyTuple_SetItem() called on an already-escaped tuple object" + raise OperationError(space.w_SystemError, space.wrap(msg)) + ref_old = ref_tup.c_items[pos] + ref_tup.c_items[pos] = ref_item # SetItem steals a reference! + Py_DecRef(space, ref_old) return 0 @cpython_api([PyObject, Py_ssize_t], PyObject) -def PyTuple_GetItem(space, w_t, pos): - if not PyTuple_Check(space, w_t): - PyErr_BadInternalCall(space) - assert isinstance(w_t, W_TupleObject) - w_obj = w_t.wrappeditems[pos] - return borrow_from(w_t, w_obj) - - at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) -def PyTuple_GET_SIZE(space, w_t): - """Return the size of the tuple p, which must be non-NULL and point to a tuple; - no error checking is performed. """ - assert isinstance(w_t, W_TupleObject) - return len(w_t.wrappeditems) +def PyTuple_GetItem(space, ref, pos): + # XXX do PyTuple_Check, without forcing ref as an interpreter object + ref_tup = rffi.cast(PyTupleObject, ref) + if ref_tup.c_items: + return Reference(ref_tup.c_items[pos]) # borrowed reference + else: + w_t = from_ref(space, ref) + w_obj = space.getitem(w_t, space.wrap(pos)) + return borrow_from(w_t, w_obj) + + at cpython_api([PyObject], Py_ssize_t, error=-1) +def _PyTuple_Size_Fast(space, ref): + # custom version: it's not a macro, so it can be called from other .py + # files; but it doesn't include PyTuple_Check() + ref_tup = rffi.cast(PyTupleObject, ref) + return ref_tup.c_size @cpython_api([PyObject], Py_ssize_t, error=-1) def PyTuple_Size(space, ref): """Take a pointer to a tuple object, and return the size of that tuple.""" - if not PyTuple_Check(space, ref): - raise OperationError(space.w_TypeError, - space.wrap("expected tuple object")) - return PyTuple_GET_SIZE(space, ref) + # XXX do PyTuple_Check, without forcing ref as an interpreter object + ref_tup = rffi.cast(PyTupleObject, ref) + return ref_tup.c_size @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyTuple_Resize(space, ref, newsize): +def _PyTuple_Resize(space, refp, newsize): """Can be used to resize a tuple. newsize will be the new length of the tuple. Because tuples are supposed to be immutable, this should only be used if there is only one reference to the object. Do not use this if the tuple may already @@ -60,18 +149,22 @@ this function. If the object referenced by *p is replaced, the original *p is destroyed. On failure, returns -1 and sets *p to NULL, and raises MemoryError or SystemError.""" - py_tuple = from_ref(space, ref[0]) - if not PyTuple_Check(space, py_tuple): - PyErr_BadInternalCall(space) - assert isinstance(py_tuple, W_TupleObject) - py_newtuple = PyTuple_New(space, newsize) - - to_cp = newsize - oldsize = len(py_tuple.wrappeditems) - if oldsize < newsize: - to_cp = oldsize - for i in range(to_cp): - py_newtuple.wrappeditems[i] = py_tuple.wrappeditems[i] - Py_DecRef(space, ref[0]) - ref[0] = make_ref(space, py_newtuple) + # XXX do PyTuple_Check, without forcing ref as an interpreter object + # XXX -- then if it fails it should reset refp[0] to null + ref_tup = rffi.cast(PyTupleObject, refp[0]) + c_newitems = lltype.malloc(ARRAY_OF_PYOBJ.TO, newsize, + flavor='raw', zero=True) + c_olditems = ref_tup.c_items + if not c_olditems: + msg = "_PyTuple_Resize() called on an already-escaped tuple object" + raise OperationError(space.w_SystemError, space.wrap(msg)) + oldsize = ref_tup.c_size + for i in range(min(oldsize, newsize)): + c_newitems[i] = c_olditems[i] + # decref items deleted by shrinkage + for i in range(newsize, oldsize): + Py_DecRef(space, c_olditems[i]) + ref_tup.c_items = c_newitems + ref_tup.c_size = newsize + lltype.free(c_olditems, flavor='raw') return 0 Modified: pypy/branch/jitypes2/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/typeobject.py Fri Dec 3 11:04:29 2010 @@ -24,7 +24,7 @@ from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, - PyNumberMethods) + PyNumberMethods, PySequenceMethods) from pypy.module.cpyext.slotdefs import ( slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) from pypy.interpreter.error import OperationError @@ -136,6 +136,8 @@ if not struct: if slot_names[0] == 'c_tp_as_number': STRUCT_TYPE = PyNumberMethods + elif slot_names[0] == 'c_tp_as_sequence': + STRUCT_TYPE = PySequenceMethods else: raise AssertionError( "Structure not allocated: %s" % (slot_names[0],)) @@ -392,6 +394,8 @@ lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') if obj_pto.c_tp_as_number: lltype.free(obj_pto.c_tp_as_number, flavor='raw') + if obj_pto.c_tp_as_sequence: + lltype.free(obj_pto.c_tp_as_sequence, flavor='raw') Py_DecRef(space, base_pyo) rffi.free_charp(obj_pto.c_tp_name) PyObject_dealloc(space, obj) @@ -433,12 +437,6 @@ finish_type_1(space, pto) finish_type_2(space, pto, w_type) - if space.type(w_type).is_cpytype(): - # XXX Types with a C metatype are never freed, try to see why... - render_immortal(pto, w_type) - lltype.render_immortal(pto) - lltype.render_immortal(pto.c_tp_name) - pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: @@ -544,25 +542,12 @@ w_obj.ready() finish_type_2(space, py_type, w_obj) - render_immortal(py_type, w_obj) state = space.fromcache(RefcountState) state.non_heaptypes_w.append(w_obj) return w_obj -def render_immortal(py_type, w_obj): - lltype.render_immortal(py_type.c_tp_bases) - lltype.render_immortal(py_type.c_tp_mro) - - assert isinstance(w_obj, W_TypeObject) - if w_obj.is_cpytype(): - lltype.render_immortal(py_type.c_tp_dict) - else: - lltype.render_immortal(py_type.c_tp_name) - if not w_obj.is_cpytype() and w_obj.is_heaptype(): - lltype.render_immortal(py_type) - def finish_type_1(space, pto): """ Sets up tp_bases, necessary before creating the interpreter type. @@ -599,7 +584,8 @@ if w_obj.is_cpytype(): Py_DecRef(space, pto.c_tp_dict) - pto.c_tp_dict = make_ref(space, w_obj.getdict()) + w_dict = space.newdict(from_strdict_shared=w_obj.dict_w) + pto.c_tp_dict = make_ref(space, w_dict) @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): Modified: pypy/branch/jitypes2/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/branch/jitypes2/pypy/module/fcntl/test/test_fcntl.py Fri Dec 3 11:04:29 2010 @@ -63,7 +63,7 @@ if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', 'Darwin1.2', 'darwin', 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', + 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', 'bsdos2', 'bsdos3', 'bsdos4', 'openbsd', 'openbsd2', 'openbsd3'): if struct.calcsize('l') == 8: @@ -159,7 +159,7 @@ if "linux" in sys.platform: TIOCGPGRP = 0x540f - elif "darwin" in sys.platform or "freebsd6" == sys.platform: + elif "darwin" in sys.platform or "freebsd" in sys.platform: TIOCGPGRP = 0x40047477 else: skip("don't know how to test ioctl() on this platform") Modified: pypy/branch/jitypes2/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/imp/importing.py (original) +++ pypy/branch/jitypes2/pypy/module/imp/importing.py Fri Dec 3 11:04:29 2010 @@ -76,7 +76,7 @@ return SEARCH_ERROR, None, None -if sys.platform in ['linux2', 'freebsd']: +if sys.platform == 'linux2' or 'freebsd' in sys.platform: def case_ok(filename): return True else: Modified: pypy/branch/jitypes2/pypy/module/itertools/interp_itertools.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/itertools/interp_itertools.py (original) +++ pypy/branch/jitypes2/pypy/module/itertools/interp_itertools.py Fri Dec 3 11:04:29 2010 @@ -391,7 +391,8 @@ self.iterators_w = iterators_w self.current_iterator = 0 self.num_iterators = len(iterators_w) - self.started = False + if self.num_iterators > 0: + self.w_it = iterators_w[0] def iter_w(self): return self.space.wrap(self) @@ -399,26 +400,23 @@ def next_w(self): if self.current_iterator >= self.num_iterators: raise OperationError(self.space.w_StopIteration, self.space.w_None) - if not self.started: - self.current_iterator = 0 - self.w_it = self.iterators_w[self.current_iterator] - self.started = True + try: + return self.space.next(self.w_it) + except OperationError, e: + return self._handle_error(e) + def _handle_error(self, e): while True: + if not e.match(self.space, self.space.w_StopIteration): + raise e + self.current_iterator += 1 + if self.current_iterator >= self.num_iterators: + raise e + self.w_it = self.iterators_w[self.current_iterator] try: - w_obj = self.space.next(self.w_it) + return self.space.next(self.w_it) except OperationError, e: - if e.match(self.space, self.space.w_StopIteration): - self.current_iterator += 1 - if self.current_iterator >= self.num_iterators: - raise OperationError(self.space.w_StopIteration, self.space.w_None) - else: - self.w_it = self.iterators_w[self.current_iterator] - else: - raise - else: - break - return w_obj + pass # loop back to the start of _handle_error(e) def W_Chain___new__(space, w_subtype, args_w): return space.wrap(W_Chain(space, args_w)) @@ -446,8 +444,10 @@ def __init__(self, space, w_fun, args_w): self.space = space - self.identity_fun = (self.space.is_w(w_fun, space.w_None)) - self.w_fun = w_fun + if self.space.is_w(w_fun, space.w_None): + self.w_fun = None + else: + self.w_fun = w_fun iterators_w = [] i = 0 @@ -470,12 +470,26 @@ return self.space.wrap(self) def next_w(self): - w_objects = self.space.newtuple([self.space.next(w_it) for w_it in self.iterators_w]) - if self.identity_fun: + # common case: 1 or 2 arguments + iterators_w = self.iterators_w + length = len(iterators_w) + if length == 1: + objects = [self.space.next(iterators_w[0])] + elif length == 2: + objects = [self.space.next(iterators_w[0]), + self.space.next(iterators_w[1])] + else: + objects = self._get_objects() + w_objects = self.space.newtuple(objects) + if self.w_fun is None: return w_objects else: return self.space.call(self.w_fun, w_objects) + def _get_objects(self): + # the loop is out of the way of the JIT + return [self.space.next(w_elem) for w_elem in self.iterators_w] + def W_IMap___new__(space, w_subtype, w_fun, args_w): if len(args_w) == 0: @@ -769,15 +783,7 @@ raise OperationError(self.space.w_StopIteration, self.space.w_None) if not self.new_group: - # Consume unwanted input until we reach the next group - try: - while True: - self.group_next(self.index) - - except StopIteration: - pass - if self.exhausted: - raise OperationError(self.space.w_StopIteration, self.space.w_None) + self._consume_unwanted_input() if not self.started: self.started = True @@ -799,6 +805,16 @@ w_iterator = self.space.wrap(W_GroupByIterator(self.space, self.index, self)) return self.space.newtuple([self.w_key, w_iterator]) + def _consume_unwanted_input(self): + # Consume unwanted input until we reach the next group + try: + while True: + self.group_next(self.index) + except StopIteration: + pass + if self.exhausted: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + def group_next(self, group_index): if group_index < self.index: raise StopIteration Modified: pypy/branch/jitypes2/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/posix/__init__.py (original) +++ pypy/branch/jitypes2/pypy/module/posix/__init__.py Fri Dec 3 11:04:29 2010 @@ -96,6 +96,8 @@ interpleveldefs['fork'] = 'interp_posix.fork' if hasattr(os, 'openpty'): interpleveldefs['openpty'] = 'interp_posix.openpty' + if hasattr(os, 'forkpty'): + interpleveldefs['forkpty'] = 'interp_posix.forkpty' if hasattr(os, 'waitpid'): interpleveldefs['waitpid'] = 'interp_posix.waitpid' if hasattr(os, 'execv'): Modified: pypy/branch/jitypes2/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/jitypes2/pypy/module/posix/interp_posix.py Fri Dec 3 11:04:29 2010 @@ -608,6 +608,14 @@ raise wrap_oserror(space, e) return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)]) +def forkpty(space): + try: + pid, master_fd = os.forkpty() + except OSError, e: + raise wrap_oserror(space, e) + return space.newtuple([space.wrap(pid), + space.wrap(master_fd)]) + def waitpid(space, pid, options): """ waitpid(pid, options) -> (pid, status) Modified: pypy/branch/jitypes2/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/jitypes2/pypy/module/posix/test/test_posix2.py Fri Dec 3 11:04:29 2010 @@ -331,6 +331,22 @@ data = os.read(master_fd, 100) assert data.startswith('x') + if hasattr(__import__(os.name), "forkpty"): + def test_forkpty(self): + import sys + os = self.posix + childpid, master_fd = os.forkpty() + assert isinstance(childpid, int) + assert isinstance(master_fd, int) + if childpid == 0: + data = os.read(0, 100) + if data.startswith('abc'): + os._exit(42) + else: + os._exit(43) + os.write(master_fd, 'abc\n') + _, status = os.waitpid(childpid, 0) + assert status >> 8 == 42 if hasattr(__import__(os.name), "execv"): def test_execv(self): Modified: pypy/branch/jitypes2/pypy/module/pyexpat/interp_pyexpat.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/pyexpat/interp_pyexpat.py (original) +++ pypy/branch/jitypes2/pypy/module/pyexpat/interp_pyexpat.py Fri Dec 3 11:04:29 2010 @@ -329,7 +329,7 @@ if self.returns_unicode: return space.call_function( space.getattr(space.wrap(s), space.wrap("decode")), - space.wrap(self.encoding), + space.wrap("utf-8"), space.wrap("strict")) else: return space.wrap(s) Modified: pypy/branch/jitypes2/pypy/module/pyexpat/test/test_parser.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/pyexpat/test/test_parser.py (original) +++ pypy/branch/jitypes2/pypy/module/pyexpat/test/test_parser.py Fri Dec 3 11:04:29 2010 @@ -15,3 +15,17 @@ assert res == 1 raises(pyexpat.ExpatError, p.Parse, "3") + + def test_encoding(self): + import pyexpat + for encoding_arg in (None, 'utf-8', 'iso-8859-1'): + for namespace_arg in (None, '{'): + print encoding_arg, namespace_arg + p = pyexpat.ParserCreate(encoding_arg, namespace_arg) + data = [] + p.CharacterDataHandler = lambda s: data.append(s) + encoding = encoding_arg is None and 'utf-8' or encoding_arg + + res = p.Parse(u"\u00f6".encode(encoding), isfinal=True) + assert res == 1 + assert data == [u"\u00f6"] Modified: pypy/branch/jitypes2/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/jitypes2/pypy/module/pypyjit/policy.py Fri Dec 3 11:04:29 2010 @@ -12,19 +12,13 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys', 'array', '_ffi']: + 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator']: return True return False def look_inside_function(self, func): - # this function should never actually return True directly - # but instead call the base implementation mod = func.__module__ or '?' - if mod.startswith('pypy.objspace.'): - # gc_id operation - if func.__name__ == 'id__ANY': - return False if mod == 'pypy.rlib.rbigint' or mod == 'pypy.rlib.rlocale': return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff Modified: pypy/branch/jitypes2/pypy/module/pypyjit/test/test_policy.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/pypyjit/test/test_policy.py (original) +++ pypy/branch/jitypes2/pypy/module/pypyjit/test/test_policy.py Fri Dec 3 11:04:29 2010 @@ -4,7 +4,7 @@ def test_id_any(): from pypy.objspace.std.default import id__ANY - assert not pypypolicy.look_inside_function(id__ANY) + assert pypypolicy.look_inside_function(id__ANY) def test_bigint(): from pypy.rlib.rbigint import rbigint Modified: pypy/branch/jitypes2/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/jitypes2/pypy/module/pypyjit/test/test_pypy_c.py Fri Dec 3 11:04:29 2010 @@ -654,6 +654,32 @@ ''', 3000, ([0], 2000*3)) assert len(self.loops) == 1 + def test_getattr_with_dynamic_attribute(self): + self.run_source(''' + class A(object): + pass + + l = ["x", "y"] + + def main(arg): + sum = 0 + a = A() + a.a1 = 0 + a.a2 = 0 + a.a3 = 0 + a.a4 = 0 + a.a5 = 0 # workaround, because the first five attributes need a promotion + a.x = 1 + a.y = 2 + i = 0 + while i < 2000: + name = l[i % 2] + sum += getattr(a, name) + i += 1 + return sum + ''', 3000, ([0], 3000)) + assert len(self.loops) == 1 + def test_blockstack_virtualizable(self): self.run_source(''' from pypyjit import residual_call @@ -697,11 +723,9 @@ i = t2[3] del t2 return i - ''', 100, ([], 100)) + ''', 40, ([], 100)) bytecode, = self.get_by_bytecode('BINARY_SUBSCR') - assert len(bytecode.get_opnames('new_array')) == 1 - # XXX I would like here to say that it's 0, but unfortunately - # call that can raise is not exchanged into getarrayitem_gc + assert len(bytecode.get_opnames('new_array')) == 0 def test_overflow_checking(self): startvalue = sys.maxint - 2147483647 Modified: pypy/branch/jitypes2/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/sys/__init__.py (original) +++ pypy/branch/jitypes2/pypy/module/sys/__init__.py Fri Dec 3 11:04:29 2010 @@ -60,8 +60,6 @@ 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', 'hexversion' : 'version.get_hexversion(space)', - 'ps1' : 'space.wrap(">>>> ")', - 'ps2' : 'space.wrap(".... ")', 'displayhook' : 'hook.displayhook', '__displayhook__' : 'hook.__displayhook__', Modified: pypy/branch/jitypes2/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py (original) +++ pypy/branch/jitypes2/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py Fri Dec 3 11:04:29 2010 @@ -11,21 +11,23 @@ py.test.skip("pypy white-box test") from _ctypes.function import CFuncPtr - guess = CFuncPtr._guess_argtypes + def guess(value): + cobj = CFuncPtr._conv_param(None, value, 0) + return type(cobj) - assert guess([13]) == [c_int] - assert guess([0]) == [c_int] - assert guess(['xca']) == [c_char_p] - assert guess([None]) == [c_void_p] - assert guess([c_int(3)]) == [c_int] - assert guess([u'xca']) == [c_wchar_p] + assert guess(13) == c_int + assert guess(0) == c_int + assert guess('xca') == c_char_p + assert guess(None) == c_void_p + assert guess(c_int(3)) == c_int + assert guess(u'xca') == c_wchar_p class Stuff: pass s = Stuff() s._as_parameter_ = None - assert guess([s]) == [c_void_p] + assert guess(s) == c_void_p def test_guess_unicode(): if not hasattr(sys, 'pypy_translation_info') and sys.platform != 'win32': Modified: pypy/branch/jitypes2/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py (original) +++ pypy/branch/jitypes2/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py Fri Dec 3 11:04:29 2010 @@ -99,7 +99,7 @@ def test_primitive(self): if not hasattr(sys, 'pypy_translation_info'): py.test.skip("pypy white-box test") - assert c_char_p("abc")._objects['0']._buffer[0] == "a" + assert c_char_p("abc")._objects._buffer[0] == "a" assert c_int(3)._objects is None def test_pointer_to_pointer(self): @@ -123,7 +123,7 @@ pass cf = CFUNCTYPE(c_int, c_int)(f) p1 = cast(cf, c_void_p) - assert p1._objects == {'1': cf, '0': {'0': cf}} + assert p1._objects == {id(cf): cf, '0': cf} def test_array_of_struct_with_pointer(self): class S(Structure): @@ -221,7 +221,7 @@ import gc; gc.collect() print 'x =', repr(x) assert x.value == 'hellohello' - assert x._objects.keys() == ['0'] + assert x._objects == 'hellohello' # class datum(Structure): _fields_ = [ Modified: pypy/branch/jitypes2/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/mapdict.py Fri Dec 3 11:04:29 2010 @@ -42,11 +42,23 @@ return None def index(self, selector): - if (self.space.config.objspace.std.withmethodcache and - not jit.we_are_jitted()): - return self._index_cache(selector) + if jit.we_are_jitted(): + # hack for the jit: + # the _index method is pure too, but its argument is never + # constant, because it is always a new tuple + return self._index_jit_pure(selector[0], selector[1]) else: - return self._index(selector) + return self._index_indirection(selector) + + @jit.purefunction + def _index_jit_pure(self, name, index): + return self._index_indirection((name, index)) + + @jit.dont_look_inside + def _index_indirection(self, selector): + if (self.space.config.objspace.std.withmethodcache): + return self._index_cache(selector) + return self._index(selector) @jit.dont_look_inside def _index_cache(self, selector): @@ -498,10 +510,11 @@ def _mapdict_read_storage(self, index): assert index >= 0 - for i in rangenmin1: - if index == i: - erased = getattr(self, "_value%s" % i) - return rerased.unerase(erased, W_Root) + if index < nmin1: + for i in rangenmin1: + if index == i: + erased = getattr(self, "_value%s" % i) + return rerased.unerase(erased, W_Root) if self._has_storage_list(): return self._mapdict_get_storage_list()[index - nmin1] erased = getattr(self, "_value%s" % nmin1) Modified: pypy/branch/jitypes2/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/objspace.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/objspace.py Fri Dec 3 11:04:29 2010 @@ -9,7 +9,7 @@ from pypy.objspace.descroperation import DescrOperation, raiseattrerror from pypy.rlib.objectmodel import instantiate, r_dict, specialize from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib.rarithmetic import base_int +from pypy.rlib.rarithmetic import base_int, widen from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.jit import hint from pypy.tool.sourcetools import func_with_new_name @@ -176,7 +176,11 @@ #print 'wrapping', x, '->', w_result return w_result if isinstance(x, base_int): - return W_LongObject.fromrarith_int(x) + x = widen(x) + if isinstance(x, int): + return self.newint(x) + else: + return W_LongObject.fromrarith_int(x) # _____ below here is where the annotator should not get _____ @@ -372,7 +376,7 @@ self, w_obj, expected_length)[:]) if expected_length != -1 and len(t) != expected_length: raise self._wrap_expected_length(expected_length, len(t)) - return t + return make_sure_not_resized(t) def fixedview_unroll(self, w_obj, expected_length=-1): return self.fixedview(w_obj, expected_length, unroll=True) Modified: pypy/branch/jitypes2/pypy/rlib/debug.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/debug.py (original) +++ pypy/branch/jitypes2/pypy/rlib/debug.py Fri Dec 3 11:04:29 2010 @@ -53,13 +53,11 @@ _log = None # patched from tests to be an object of class DebugLog # or compatible -_stderr = sys.stderr # alternatively, this is patched from tests - # (redirects debug_print(), but not debug_start/stop) def debug_print(*args): for arg in args: - print >> _stderr, arg, - print >> _stderr + print >> sys.stderr, arg, + print >> sys.stderr if _log is not None: _log.debug_print(*args) @@ -87,13 +85,15 @@ _stop_colors = "" def debug_start(category): - print >> sys.stderr, '%s[%s] {%s%s' % (_start_colors_1, time.clock(), + c = int(time.clock() * 100) + print >> sys.stderr, '%s[%x] {%s%s' % (_start_colors_1, c, category, _stop_colors) if _log is not None: _log.debug_start(category) def debug_stop(category): - print >> sys.stderr, '%s[%s] %s}%s' % (_start_colors_2, time.clock(), + c = int(time.clock() * 100) + print >> sys.stderr, '%s[%x] %s}%s' % (_start_colors_2, c, category, _stop_colors) if _log is not None: _log.debug_stop(category) @@ -226,31 +226,6 @@ hop.exception_cannot_occur() return hop.inputarg(hop.args_r[0], arg=0) -def make_sure_not_modified(arg): - """ Function checking whether annotation of SomeList is never resized - and never modified, useful for debugging. Does nothing when run directly - """ - return arg - -class Entry(ExtRegistryEntry): - _about_ = make_sure_not_modified - - def compute_result_annotation(self, s_arg): - from pypy.annotation.model import SomeList - assert isinstance(s_arg, SomeList) - # the logic behind it is that we try not to propagate - # make_sure_not_resized, when list comprehension is not on - if self.bookkeeper.annotator.translator.config.translation.list_comprehension_operations: - s_arg.listdef.never_mutate() - else: - from pypy.annotation.annrpython import log - log.WARNING('make_sure_not_modified called, but has no effect since list_comprehension is off') - return s_arg - - def specialize_call(self, hop): - hop.exception_cannot_occur() - return hop.inputarg(hop.args_r[0], arg=0) - class IntegerCanBeNegative(Exception): pass Modified: pypy/branch/jitypes2/pypy/rlib/jit.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/jit.py (original) +++ pypy/branch/jitypes2/pypy/rlib/jit.py Fri Dec 3 11:04:29 2010 @@ -156,7 +156,7 @@ def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1, arg3=-sys.maxint-1, arg4=-sys.maxint-1): - """When JITted, cause an extra operation DEBUG_MERGE_POINT to appear in + """When JITted, cause an extra operation JIT_DEBUG to appear in the graphs. Should not be left after debugging.""" keepalive_until_here(string) # otherwise the whole function call is removed jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)' @@ -260,17 +260,12 @@ OPTIMIZER_NO_PERFECTSPEC = 1 OPTIMIZER_FULL = 2 -DEBUG_OFF = 0 -DEBUG_PROFILE = 1 -DEBUG_STEPS = 2 -DEBUG_DETAILED = 3 - PARAMETERS = {'threshold': 1000, 'trace_eagerness': 200, 'trace_limit': 10000, 'inlining': False, 'optimizer': OPTIMIZER_FULL, - 'debug' : DEBUG_STEPS, + 'loop_longevity': 1000, } unroll_parameters = unrolling_iterable(PARAMETERS.keys()) Modified: pypy/branch/jitypes2/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/libffi.py (original) +++ pypy/branch/jitypes2/pypy/rlib/libffi.py Fri Dec 3 11:04:29 2010 @@ -200,6 +200,9 @@ # the optimizer will fail to recognize the pattern and won't turn it # into a fast CALL. Note that "arg = arg.next" is optimized away, # assuming that archain is completely virtual. + if argchain.numargs != len(self.argtypes): + raise TypeError, 'Wrong number of arguments: %d expected, got %d' %\ + (argchain.numargs, len(self.argtypes)) ll_args = self._prepare() i = 0 arg = argchain.first Modified: pypy/branch/jitypes2/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/jitypes2/pypy/rlib/rarithmetic.py Fri Dec 3 11:04:29 2010 @@ -92,7 +92,8 @@ return False r_class = rffi.platform.numbertype_to_rclass[tp] assert issubclass(r_class, base_int) - return r_class.BITS < LONG_BIT + return r_class.BITS < LONG_BIT or ( + r_class.BITS == LONG_BIT and r_class.SIGNED) _should_widen_type._annspecialcase_ = 'specialize:memo' del _bits, _itest, _Ltest @@ -389,6 +390,11 @@ r_longlong = build_int('r_longlong', True, 64) r_ulonglong = build_int('r_ulonglong', False, 64) +if r_longlong is not r_int: + r_int64 = r_longlong +else: + r_int64 = int + # float as string -> sign, beforept, afterpt, exponent Modified: pypy/branch/jitypes2/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/rdynload.py (original) +++ pypy/branch/jitypes2/pypy/rlib/rdynload.py Fri Dec 3 11:04:29 2010 @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD_7 = platform.name == "freebsd7" +_FREEBSD = platform.name == "freebsd" if _WIN32: from pypy.rlib import rwin32 @@ -27,7 +27,7 @@ else: pre_include_bits = [] -if _FREEBSD_7 or _WIN32: +if _FREEBSD or _WIN32: libraries = [] else: libraries = ['dl'] Modified: pypy/branch/jitypes2/pypy/rlib/rerased.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/rerased.py (original) +++ pypy/branch/jitypes2/pypy/rlib/rerased.py Fri Dec 3 11:04:29 2010 @@ -91,7 +91,7 @@ return annmodel.SomeInteger() assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() return annmodel.SomeInstance(clsdef) def specialize_call(self, hop): @@ -108,7 +108,7 @@ def compute_result_annotation(self, s_obj, s_type): assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() s_item = annmodel.SomeInstance(clsdef) return self.bookkeeper.newlist(s_item) Modified: pypy/branch/jitypes2/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/jitypes2/pypy/rlib/rsre/rsre_core.py Fri Dec 3 11:04:29 2010 @@ -1,5 +1,5 @@ import sys -from pypy.rlib.debug import check_nonneg, make_sure_not_modified +from pypy.rlib.debug import check_nonneg from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rsre import rsre_char from pypy.tool.sourcetools import func_with_new_name @@ -471,7 +471,6 @@ while True: op = ctx.pat(ppos) ppos += 1 - make_sure_not_modified(ctx.pattern) #jit.jit_debug("sre_match", op, ppos, ptr) # Modified: pypy/branch/jitypes2/pypy/rlib/rsre/test/test_zjit.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/rsre/test/test_zjit.py (original) +++ pypy/branch/jitypes2/pypy/rlib/rsre/test/test_zjit.py Fri Dec 3 11:04:29 2010 @@ -1,6 +1,5 @@ from pypy.jit.metainterp.test import test_basic from pypy.rlib.nonconst import NonConstant -from pypy.rlib.debug import make_sure_not_modified from pypy.rlib.rsre.test.test_match import get_code from pypy.rlib.rsre import rsre_core from pypy.rpython.lltypesystem import lltype Modified: pypy/branch/jitypes2/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/jitypes2/pypy/rlib/test/test_debug.py Fri Dec 3 11:04:29 2010 @@ -42,14 +42,14 @@ py.test.raises(IntegerCanBeNegative, interpret, g, [9]) def test_make_sure_not_resized(): - from pypy.annotation.listdef import TooLateForChange + from pypy.annotation.listdef import ListChangeUnallowed def f(): result = [1,2,3] make_sure_not_resized(result) result.append(4) return len(result) - py.test.raises(TooLateForChange, interpret, f, [], + py.test.raises(ListChangeUnallowed, interpret, f, [], list_comprehension_operations=True) Modified: pypy/branch/jitypes2/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/test/test_libffi.py (original) +++ pypy/branch/jitypes2/pypy/rlib/test/test_libffi.py Fri Dec 3 11:04:29 2010 @@ -282,3 +282,24 @@ res = self.call(func, [x, y], rffi.FLOAT, init_result=0.0) expected = c_float(c_float(12.34).value + c_float(56.78).value).value assert res == expected + + def test_wrong_number_of_arguments(self): + from pypy.rpython.llinterp import LLException + libfoo = self.get_libfoo() + func = (libfoo, 'sum_xy', [types.sint, types.double], types.sint) + + glob = globals() + loc = locals() + def my_raises(s): + try: + exec s in glob, loc + except TypeError: + pass + except LLException, e: + if str(e) != "": + raise + else: + assert False, 'Did not raise' + + my_raises("self.call(func, [38], rffi.LONG)") # one less + my_raises("self.call(func, [38, 12.3, 42], rffi.LONG)") # one more Modified: pypy/branch/jitypes2/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/extfunc.py (original) +++ pypy/branch/jitypes2/pypy/rpython/extfunc.py Fri Dec 3 11:04:29 2010 @@ -201,6 +201,11 @@ exec py.code.compile(""" from pypy.rlib.objectmodel import running_on_llinterp from pypy.rlib.debug import llinterpcall + from pypy.rlib.jit import dont_look_inside + # note: we say 'dont_look_inside' mostly because the + # JIT does not support 'running_on_llinterp', but in + # theory it is probably right to stop jitting anyway. + @dont_look_inside def ll_wrapper(%s): if running_on_llinterp: return llinterpcall(s_result, fakeimpl, %s) Modified: pypy/branch/jitypes2/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/jitypes2/pypy/rpython/lltypesystem/ll2ctypes.py Fri Dec 3 11:04:29 2010 @@ -442,6 +442,9 @@ "not allocated from RPython at all") self._storage = None + def _getid(self): + return self._addressof_storage() + def __eq__(self, other): if isinstance(other, _llgcopaque): addressof_other = other.intval @@ -1291,7 +1294,7 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7'): + elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() Modified: pypy/branch/jitypes2/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/jitypes2/pypy/rpython/lltypesystem/llmemory.py Fri Dec 3 11:04:29 2010 @@ -93,8 +93,10 @@ return endmarker._as_ptr() else: return parent.getitem(index)._as_ptr() - elif (isinstance(A, lltype.FixedSizeArray) and - array_item_type_match(A.OF, self.TYPE)): + elif ((isinstance(A, lltype.FixedSizeArray) + or (isinstance(A, lltype.Array) and A._hints.get('nolength', + False))) + and array_item_type_match(A.OF, self.TYPE)): # for array of primitives or pointers return lltype.direct_ptradd(firstitemptr, self.repeat) else: Modified: pypy/branch/jitypes2/pypy/rpython/lltypesystem/rbuiltin.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/lltypesystem/rbuiltin.py (original) +++ pypy/branch/jitypes2/pypy/rpython/lltypesystem/rbuiltin.py Fri Dec 3 11:04:29 2010 @@ -42,7 +42,7 @@ return hop.genop('cast_pointer', [v_inst], # v_type implicit in r_result resulttype = hop.r_result.lowleveltype) - classdef = s_class.descriptions.keys()[0].getuniqueclassdef() + classdef = s_class.any_description().getuniqueclassdef() return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops) def rtype_builtin_hasattr(hop): Modified: pypy/branch/jitypes2/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/jitypes2/pypy/rpython/lltypesystem/rclass.py Fri Dec 3 11:04:29 2010 @@ -392,7 +392,7 @@ source_classdef = source_desc.getclassdef(None) source_repr = getinstancerepr(self.rtyper, source_classdef) assert len(s_func.descriptions) == 1 - funcdesc = s_func.descriptions.keys()[0] + funcdesc, = s_func.descriptions graph = funcdesc.getuniquegraph() FUNCTYPE = FuncType([Ptr(source_repr.object_type)], Void) destrptr = functionptr(FUNCTYPE, graph.name, Modified: pypy/branch/jitypes2/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/jitypes2/pypy/rpython/lltypesystem/rdict.py Fri Dec 3 11:04:29 2010 @@ -581,7 +581,7 @@ def ll_dict_lookup_clean(d, hash): # a simplified version of ll_dict_lookup() which assumes that the # key is new, and the dictionary doesn't contain deleted entries. - # It only find the next free slot for the given hash. + # It only finds the next free slot for the given hash. entries = d.entries mask = len(entries) - 1 i = hash & mask Modified: pypy/branch/jitypes2/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/branch/jitypes2/pypy/rpython/lltypesystem/rpbc.py Fri Dec 3 11:04:29 2010 @@ -127,7 +127,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(self.rtyper, self.callfamily) assert len(uniquerows) == 1 @@ -166,7 +166,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) def convert_desc(self, funcdesc): @@ -230,7 +230,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/base.py Fri Dec 3 11:04:29 2010 @@ -5,7 +5,6 @@ from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rlib.rarithmetic import r_uint TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed), ('size', lltype.Signed), @@ -19,6 +18,7 @@ malloc_zero_filled = False prebuilt_gc_objects_are_static_roots = True object_minimal_size = 0 + gcflag_extra = 0 # or a real GC flag that is always 0 when not collecting def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, translated_to_c=True): @@ -410,42 +410,6 @@ GCClass = getattr(module, classname) return GCClass, GCClass.TRANSLATION_PARAMS -def _read_float_and_factor_from_env(varname): - import os - value = os.environ.get(varname) - if value: - if len(value) > 1 and value[-1] in 'bB': - value = value[:-1] - realvalue = value[:-1] - if value[-1] in 'kK': - factor = 1024 - elif value[-1] in 'mM': - factor = 1024*1024 - elif value[-1] in 'gG': - factor = 1024*1024*1024 - else: - factor = 1 - realvalue = value - try: - return (float(realvalue), factor) - except ValueError: - pass - return (0.0, 0) - -def read_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - return int(value * factor) - -def read_uint_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - return r_uint(value * factor) - -def read_float_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - if factor != 1: - return 0.0 - return value - def _convert_callback_formats(callback): callback = getattr(callback, 'im_func', callback) if callback not in _converted_callback_formats: Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/generation.py Fri Dec 3 11:04:29 2010 @@ -2,7 +2,7 @@ from pypy.rpython.memory.gc.semispace import SemiSpaceGC from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL, GCFLAG_FORWARDED from pypy.rpython.memory.gc.semispace import GC_HASH_TAKEN_ADDR -from pypy.rpython.memory.gc.base import read_from_env +from pypy.rpython.memory.gc import env from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object @@ -93,7 +93,7 @@ if self.auto_nursery_size: newsize = nursery_size_from_env() if newsize <= 0: - newsize = estimate_best_nursery_size() + newsize = env.estimate_best_nursery_size() if newsize > 0: self.set_nursery_size(newsize) @@ -633,139 +633,5 @@ # ____________________________________________________________ -import os - def nursery_size_from_env(): - return read_from_env('PYPY_GENERATIONGC_NURSERY') - -def best_nursery_size_for_L2cache(L2cache): - # Heuristically, the best nursery size to choose is about half - # of the L2 cache. XXX benchmark some more. - return L2cache // 2 - - -if sys.platform == 'linux2': - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. - """ - debug_start("gc-L2cache") - L2cache = sys.maxint - try: - fd = os.open('/proc/cpuinfo', os.O_RDONLY, 0644) - try: - data = [] - while True: - buf = os.read(fd, 4096) - if not buf: - break - data.append(buf) - finally: - os.close(fd) - except OSError: - pass - else: - data = ''.join(data) - linepos = 0 - while True: - start = findend(data, '\ncache size', linepos) - if start < 0: - break # done - linepos = findend(data, '\n', start) - if linepos < 0: - break # no end-of-line?? - # *** data[start:linepos] == " : 2048 KB\n" - start = skipspace(data, start) - if data[start] != ':': - continue - # *** data[start:linepos] == ": 2048 KB\n" - start = skipspace(data, start + 1) - # *** data[start:linepos] == "2048 KB\n" - end = start - while '0' <= data[end] <= '9': - end += 1 - # *** data[start:end] == "2048" - if start == end: - continue - number = int(data[start:end]) - # *** data[end:linepos] == " KB\n" - end = skipspace(data, end) - if data[end] not in ('K', 'k'): # assume kilobytes for now - continue - number = number * 1024 - # for now we look for the smallest of the L2 caches of the CPUs - if number < L2cache: - L2cache = number - - debug_print("L2cache =", L2cache) - debug_stop("gc-L2cache") - - if L2cache < sys.maxint: - return best_nursery_size_for_L2cache(L2cache) - else: - # Print a top-level warning even in non-debug builds - llop.debug_print(lltype.Void, - "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") - return -1 - - def findend(data, pattern, pos): - pos = data.find(pattern, pos) - if pos < 0: - return -1 - return pos + len(pattern) - - def skipspace(data, pos): - while data[pos] in (' ', '\t'): - pos += 1 - return pos - -elif sys.platform == 'darwin': - from pypy.rpython.lltypesystem import rffi - - sysctlbyname = rffi.llexternal('sysctlbyname', - [rffi.CCHARP, rffi.VOIDP, rffi.SIZE_TP, - rffi.VOIDP, rffi.SIZE_T], - rffi.INT, - sandboxsafe=True) - - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. - """ - debug_start("gc-L2cache") - L2cache = 0 - l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') - try: - len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') - try: - size = rffi.sizeof(rffi.LONGLONG) - l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) - len_p[0] = rffi.cast(rffi.SIZE_T, size) - # XXX a hack for llhelper not being robust-enough - result = sysctlbyname("hw.l2cachesize", - rffi.cast(rffi.VOIDP, l2cache_p), - len_p, - lltype.nullptr(rffi.VOIDP.TO), - rffi.cast(rffi.SIZE_T, 0)) - if (rffi.cast(lltype.Signed, result) == 0 and - rffi.cast(lltype.Signed, len_p[0]) == size): - L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) - if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: - L2cache = 0 # overflow! - finally: - lltype.free(len_p, flavor='raw') - finally: - lltype.free(l2cache_p, flavor='raw') - debug_print("L2cache =", L2cache) - debug_stop("gc-L2cache") - if L2cache > 0: - return best_nursery_size_for_L2cache(L2cache) - else: - # Print a top-level warning even in non-debug builds - llop.debug_print(lltype.Void, - "Warning: cannot find your CPU L2 cache size with sysctl()") - return -1 - -else: - def estimate_best_nursery_size(): - return -1 # XXX implement me for other platforms + return env.read_from_env('PYPY_GENERATIONGC_NURSERY') Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/inspector.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/inspector.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/inspector.py Fri Dec 3 11:04:29 2010 @@ -101,21 +101,24 @@ AddressStack = get_address_stack() -class HeapDumper: +class HeapDumper(object): _alloc_flavor_ = "raw" BUFSIZE = 8192 # words def __init__(self, gc, fd): self.gc = gc + self.gcflag = gc.gcflag_extra self.fd = rffi.cast(rffi.INT, fd) self.writebuffer = lltype.malloc(rffi.LONGP.TO, self.BUFSIZE, flavor='raw') self.buf_count = 0 - self.seen = AddressDict() + if self.gcflag == 0: + self.seen = AddressDict() self.pending = AddressStack() def delete(self): - self.seen.delete() + if self.gcflag == 0: + self.seen.delete() self.pending.delete() lltype.free(self.writebuffer, flavor='raw') free_non_gc_object(self) @@ -140,6 +143,8 @@ self.flush() write._always_inline_ = True + # ---------- + def write_marker(self): self.write(0) self.write(0) @@ -161,9 +166,15 @@ self.add(obj) def add(self, obj): - if not self.seen.contains(obj): - self.seen.setitem(obj, obj) - self.pending.append(obj) + if self.gcflag == 0: + if not self.seen.contains(obj): + self.seen.setitem(obj, obj) + self.pending.append(obj) + else: + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) == 0: + hdr.tid |= self.gcflag + self.pending.append(obj) def add_roots(self): self.gc.enumerate_all_roots(_hd_add_root, self) @@ -177,14 +188,50 @@ while pending.non_empty(): self.writeobj(pending.pop()) + # ---------- + # A simplified copy of the above, to make sure we walk again all the + # objects to clear the 'gcflag'. + + def unwriteobj(self, obj): + gc = self.gc + gc.trace(obj, self._unwriteref, None) + + def _unwriteref(self, pointer, _): + obj = pointer.address[0] + self.unadd(obj) + + def unadd(self, obj): + assert self.gcflag != 0 + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) != 0: + hdr.tid &= ~self.gcflag + self.pending.append(obj) + + def clear_gcflag_again(self): + self.gc.enumerate_all_roots(_hd_unadd_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.unwalk(pendingroots) + pendingroots.delete() + + def unwalk(self, pending): + while pending.non_empty(): + self.unwriteobj(pending.pop()) + def _hd_add_root(obj, heap_dumper): heap_dumper.add(obj) +def _hd_unadd_root(obj, heap_dumper): + heap_dumper.unadd(obj) + def dump_rpy_heap(gc, fd): heapdumper = HeapDumper(gc, fd) heapdumper.add_roots() heapdumper.walk(heapdumper.pending) heapdumper.flush() + if heapdumper.gcflag != 0: + heapdumper.clear_gcflag_again() + heapdumper.unwalk(heapdumper.pending) heapdumper.delete() return True Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/markcompact.py Fri Dec 3 11:04:29 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup -from pypy.rpython.memory.gc.base import MovingGCBase, read_from_env +from pypy.rpython.memory.gc.base import MovingGCBase +from pypy.rpython.memory.gc import env from pypy.rlib.debug import ll_assert, have_debug_prints from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.support import get_address_stack, get_address_deque @@ -110,10 +111,10 @@ return next def setup(self): - envsize = read_from_env('PYPY_MARKCOMPACTGC_MAX') + envsize = env.read_from_env('PYPY_MARKCOMPACTGC_MAX') if envsize >= 4096: self.space_size = envsize & ~4095 - mincollect = read_from_env('PYPY_MARKCOMPACTGC_MIN') + mincollect = env.read_from_env('PYPY_MARKCOMPACTGC_MIN') if mincollect >= 4096: self.min_next_collect_after = mincollect Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/minimark.py Fri Dec 3 11:04:29 2010 @@ -12,7 +12,7 @@ collection. PYPY_GC_GROWTH Major collection threshold's max growth rate. - Default is '1.3'. Useful to collect more often + Default is '1.4'. Useful to collect more often than normally on sudden memory growth, e.g. when there is a temporary peak in memory usage. @@ -22,6 +22,12 @@ crash the program with a fatal error. Try values like '1.6GB'. + PYPY_GC_MAX_DELTA The major collection threshold will never be set + to more than PYPY_GC_MAX_DELTA the amount really + used after a collection. Defaults to 1/8th of the + total RAM size (which is constrained to be at most + 2/3/4GB on 32-bit systems). Try values like '200MB'. + PYPY_GC_MIN Don't collect while the memory size is below this limit. Useful to avoid spending all the time in the GC in very small programs. Defaults to 8 @@ -36,7 +42,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage from pypy.rpython.memory.gc.base import GCBase, MovingGCBase -from pypy.rpython.memory.gc import minimarkpage, base, generation +from pypy.rpython.memory.gc import minimarkpage, env from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint from pypy.rlib.rarithmetic import LONG_BIT_SHIFT from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop @@ -96,6 +102,7 @@ needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False malloc_zero_filled = True # xxx experiment with False + gcflag_extra = GCFLAG_FINALIZATION_ORDERING # All objects start with a HDR, i.e. with a field 'tid' which contains # a word. This word is divided in two halves: the lower half contains @@ -154,7 +161,7 @@ # grow at most by the following factor from one collection to the # next. Used e.g. when there is a sudden, temporary peak in memory # usage; this avoids that the upper bound grows too fast. - "growth_rate_max": 1.3, + "growth_rate_max": 1.4, # The number of array indices that are mapped to a single bit in # write_barrier_from_array(). Must be a power of two. The default @@ -198,6 +205,7 @@ self.min_heap_size = 0.0 self.max_heap_size = 0.0 self.max_heap_size_already_raised = False + self.max_delta = float(r_uint(-1)) # self.card_page_indices = card_page_indices if self.card_page_indices > 0: @@ -282,36 +290,42 @@ # # From there on, the GC is fully initialized and the code # below can use it - newsize = base.read_from_env('PYPY_GC_NURSERY') + newsize = env.read_from_env('PYPY_GC_NURSERY') # PYPY_GC_NURSERY=1 forces a minor collect for every malloc. # Useful to debug external factors, like trackgcroot or the # handling of the write barrier. self.debug_always_do_minor_collect = newsize == 1 if newsize <= 0: - newsize = generation.estimate_best_nursery_size() + newsize = env.estimate_best_nursery_size() if newsize <= 0: newsize = defaultsize newsize = max(newsize, minsize) # - major_coll = base.read_float_from_env('PYPY_GC_MAJOR_COLLECT') + major_coll = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT') if major_coll > 1.0: self.major_collection_threshold = major_coll # - growth = base.read_float_from_env('PYPY_GC_GROWTH') + growth = env.read_float_from_env('PYPY_GC_GROWTH') if growth > 1.0: self.growth_rate_max = growth # - min_heap_size = base.read_uint_from_env('PYPY_GC_MIN') + min_heap_size = env.read_uint_from_env('PYPY_GC_MIN') if min_heap_size > 0: self.min_heap_size = float(min_heap_size) else: # defaults to 8 times the nursery self.min_heap_size = newsize * 8 # - max_heap_size = base.read_uint_from_env('PYPY_GC_MAX') + max_heap_size = env.read_uint_from_env('PYPY_GC_MAX') if max_heap_size > 0: self.max_heap_size = float(max_heap_size) # + max_delta = env.read_uint_from_env('PYPY_GC_MAX_DELTA') + if max_delta > 0: + self.max_delta = float(max_delta) + else: + self.max_delta = 0.125 * env.get_total_memory() + # self.minor_collection() # to empty the nursery llarena.arena_free(self.nursery) self.nursery_size = newsize @@ -638,8 +652,13 @@ # means recording that they have a smaller size, so that when # moved out of the nursery, they will consume less memory. # In particular, an array with GCFLAG_HAS_CARDS is never resized. + # Also, a nursery object with GCFLAG_HAS_SHADOW is not resized + # either, as this would potentially loose part of the memory in + # the already-allocated shadow. if not self.is_in_nursery(obj): return False + if self.header(obj).tid & GCFLAG_HAS_SHADOW: + return False # size_gc_header = self.gcheaderbuilder.size_gc_header typeid = self.get_type_id(obj) @@ -1245,9 +1264,12 @@ # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than + # we currently have -- but no more than 'max_delta' more than # we currently have. + total_memory_used = float(self.get_total_memory_used()) bounded = self.set_major_threshold_from( - self.get_total_memory_used() * self.major_collection_threshold, + min(total_memory_used * self.major_collection_threshold, + total_memory_used + self.max_delta), reserving_size) # # Max heap size: gives an upper bound on the threshold. If we @@ -1406,12 +1428,21 @@ size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + size) - # initialize to an invalid tid *without* GCFLAG_VISITED, - # so that if the object dies before the next minor - # collection, the shadow will stay around but be collected - # by the next major collection. + # Initialize the shadow enough to be considered a + # valid gc object. If the original object stays + # alive at the next minor collection, it will anyway + # be copied over the shadow and overwrite the + # following fields. But if the object dies, then + # the shadow will stay around and only be freed at + # the next major collection, at which point we want + # it to look valid (but ready to be freed). shadow = shadowhdr + size_gc_header - self.header(shadow).tid = 0 + self.header(shadow).tid = self.header(obj).tid + typeid = self.get_type_id(obj) + if self.is_varsize(typeid): + lenofs = self.varsize_offset_to_length(typeid) + (shadow + lenofs).signed[0] = (obj + lenofs).signed[0] + # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.young_objects_shadows.setitem(obj, shadow) # Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/semispace.py Fri Dec 3 11:04:29 2010 @@ -42,6 +42,7 @@ inline_simple_malloc_varsize = True malloc_zero_filled = True first_unused_gcflag = first_gcflag << 5 + gcflag_extra = GCFLAG_FINALIZATION_ORDERING HDR = lltype.Struct('header', ('tid', lltype.Signed)) # XXX or rffi.INT? typeid_is_in_field = 'tid' Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/test/test_direct.py Fri Dec 3 11:04:29 2010 @@ -60,7 +60,7 @@ pass -class DirectGCTest(object): +class BaseDirectGCTest(object): GC_PARAMS = {} def setup_method(self, meth): @@ -106,6 +106,9 @@ addr = self.gc.malloc(self.get_type_id(TYPE), n, zero=True) return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) + +class DirectGCTest(BaseDirectGCTest): + def test_simple(self): p = self.malloc(S) p.x = 5 @@ -339,6 +342,15 @@ self.gc.collect() assert hash == self.gc.identityhash(self.stackroots[-1]) self.stackroots.pop() + # (7) the same, but the objects are dying young + for i in range(10): + self.gc.collect() + p = self.malloc(VAR, i) + self.stackroots.append(p) + hash1 = self.gc.identityhash(p) + hash2 = self.gc.identityhash(p) + assert hash1 == hash2 + self.stackroots.pop() def test_memory_alignment(self): A1 = lltype.GcArray(lltype.Char) Modified: pypy/branch/jitypes2/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gctransform/asmgcroot.py Fri Dec 3 11:04:29 2010 @@ -139,12 +139,13 @@ self._shape_decompressor = ShapeDecompressor() if hasattr(gctransformer.translator, '_jit2gc'): jit2gc = gctransformer.translator._jit2gc - self._extra_gcmapstart = jit2gc['gcmapstart'] - self._extra_gcmapend = jit2gc['gcmapend'] + self._extra_gcmapstart = jit2gc['gcmapstart'] + self._extra_gcmapend = jit2gc['gcmapend'] + self._extra_mark_sorted = jit2gc['gcmarksorted'] else: - returns_null = lambda: llmemory.NULL - self._extra_gcmapstart = returns_null - self._extra_gcmapend = returns_null + self._extra_gcmapstart = lambda: llmemory.NULL + self._extra_gcmapend = lambda: llmemory.NULL + self._extra_mark_sorted = lambda: True def need_thread_support(self, gctransformer, getfn): # Threads supported "out of the box" by the rest of the code. @@ -295,14 +296,16 @@ # we have a non-empty JIT-produced table to look in item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) if item: - self._shape_decompressor.setaddr(item.address[1]) + self._shape_decompressor.setaddr(item) return # maybe the JIT-produced table is not sorted? - sort_gcmap(gcmapstart2, gcmapend2) - item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) - if item: - self._shape_decompressor.setaddr(item.address[1]) - return + was_already_sorted = self._extra_mark_sorted() + if not was_already_sorted: + sort_gcmap(gcmapstart2, gcmapend2) + item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) + if item: + self._shape_decompressor.setaddr(item) + return # the item may have been not found because the main array was # not sorted. Sort it and try again. win32_follow_gcmap_jmp(gcmapstart, gcmapend) @@ -357,7 +360,8 @@ The interval from the start address (included) to the end address (excluded) is assumed to be a sorted arrays of pairs (addr1, addr2). This searches for the item with a given addr1 and returns its - address. + address. If not found exactly, it tries to return the address + of the item left of addr1 (i.e. such that result.address[0] < addr1). """ count = (end - start) // arrayitemsize while count > 1: @@ -386,7 +390,7 @@ # (item.signed[1] is an address in this case, not a signed at all!) item = binary_search(gcmapstart, gcmapend, retaddr) if item.address[0] == retaddr: - return item # found + return item.address[1] # found else: return llmemory.NULL # failed Modified: pypy/branch/jitypes2/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gctransform/boehm.py Fri Dec 3 11:04:29 2010 @@ -120,10 +120,11 @@ fptr = self.annotate_finalizer(d['ll_finalizer'], [llmemory.Address], lltype.Void) elif destrptr: EXC_INSTANCE_TYPE = self.translator.rtyper.exceptiondata.lltype_of_exception_value + typename = TYPE.__name__ def ll_finalizer(addr): exc_instance = llop.gc_fetch_exception(EXC_INSTANCE_TYPE) v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) llop.gc_restore_exception(lltype.Void, exc_instance) fptr = self.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) else: Modified: pypy/branch/jitypes2/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gctransform/framework.py Fri Dec 3 11:04:29 2010 @@ -1204,9 +1204,10 @@ assert not type_contains_pyobjs(TYPE), "not implemented" if destrptr: + typename = TYPE.__name__ def ll_finalizer(addr): v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) fptr = self.transformer.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) Modified: pypy/branch/jitypes2/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gctransform/refcounting.py Fri Dec 3 11:04:29 2010 @@ -227,7 +227,7 @@ # refcount is at zero, temporarily bump it to 1: gcheader.refcount = 1 destr_v = cast_pointer(DESTR_ARG, v) - ll_call_destructor(destrptr, destr_v) + ll_call_destructor(destrptr, destr_v, %r) refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: @@ -239,7 +239,7 @@ pop_alive(exc_instance) # XXX layering of exceptiontransform versus gcpolicy -""" % (body, TYPE._gckind) +""" % (TYPE.__name__, body, TYPE._gckind) else: call_del = None body = '\n'.join(_static_deallocator_body_for_type('v', TYPE)) Modified: pypy/branch/jitypes2/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gctransform/support.py Fri Dec 3 11:04:29 2010 @@ -98,11 +98,15 @@ hop.exception_cannot_occur() return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const) -def ll_call_destructor(destrptr, destr_v): +def ll_call_destructor(destrptr, destr_v, typename): try: destrptr(destr_v) - except: + except Exception, e: try: - os.write(2, "a destructor raised an exception, ignoring it\n") + os.write(2, "a destructor of type ") + os.write(2, typename) + os.write(2, " raised an exception ") + os.write(2, str(e)) + os.write(2, " ignoring it\n") except: pass Modified: pypy/branch/jitypes2/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/jitypes2/pypy/rpython/module/ll_os.py Fri Dec 3 11:04:29 2010 @@ -1386,6 +1386,25 @@ return extdef([], (int, int), "ll_os.ll_os_openpty", llimpl=openpty_llimpl) + @registering_if(os, 'forkpty') + def register_os_forkpty(self): + os_forkpty = self.llexternal( + 'forkpty', + [rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP], + rffi.PID_T, + compilation_info=ExternalCompilationInfo(libraries=['util'])) + def forkpty_llimpl(): + master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + childpid = os_forkpty(master_p, None, None, None) + master_fd = master_p[0] + lltype.free(master_p, flavor='raw') + if childpid == -1: + raise OSError(rposix.get_errno(), "os_forkpty failed") + return (rffi.cast(lltype.Signed, childpid), + rffi.cast(lltype.Signed, master_fd)) + + return extdef([], (int, int), "ll_os.ll_os_forkpty", + llimpl=forkpty_llimpl) @registering(os._exit) def register_os__exit(self): Modified: pypy/branch/jitypes2/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/module/ll_time.py (original) +++ pypy/branch/jitypes2/pypy/rpython/module/ll_time.py Fri Dec 3 11:04:29 2010 @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if sys.platform == 'freebsd7': +if "freebsd" in sys.platform: libraries = ['compat'] else: libraries = [] Modified: pypy/branch/jitypes2/pypy/rpython/ootypesystem/rpbc.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/ootypesystem/rpbc.py (original) +++ pypy/branch/jitypes2/pypy/rpython/ootypesystem/rpbc.py Fri Dec 3 11:04:29 2010 @@ -49,7 +49,7 @@ return hop.genop('runtimenew', [v_class], resulttype=resulttype) def getlowleveltype(self): - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) # if any of the classdefs get the lowleveltype ootype.Class, # we can only pick ootype.Class for us too. Otherwise META. for classdesc in classdescs: @@ -70,7 +70,7 @@ class MethodImplementations(object): def __init__(self, rtyper, methdescs): - samplemdesc = methdescs.iterkeys().next() + samplemdesc = iter(methdescs).next() concretetable, uniquerows = get_concrete_calltable(rtyper, samplemdesc.funcdesc.getcallfamily()) self.row_mapping = {} @@ -117,7 +117,7 @@ concretetable = None # set by _setup_repr_final def _setup_repr_final(self): - sampledesc = self.s_pbc.descriptions.iterkeys().next() + sampledesc = self.s_pbc.any_description() self.concretetable, _ = get_concrete_calltable(self.rtyper, sampledesc.funcdesc.getcallfamily()) Modified: pypy/branch/jitypes2/pypy/rpython/rlist.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/rlist.py (original) +++ pypy/branch/jitypes2/pypy/rpython/rlist.py Fri Dec 3 11:04:29 2010 @@ -9,7 +9,7 @@ from pypy.rpython import robject from pypy.rlib.objectmodel import malloc_zero_filled from pypy.rlib.debug import ll_assert -from pypy.rlib.rarithmetic import ovfcheck, widen +from pypy.rlib.rarithmetic import ovfcheck, widen, r_uint, intmask from pypy.rpython.annlowlevel import ADTInterface from pypy.rlib import rgc @@ -241,17 +241,22 @@ class __extend__(pairtype(AbstractBaseListRepr, IntegerRepr)): def rtype_getitem((r_lst, r_int), hop, checkidx=False): + v_lst, v_index = hop.inputargs(r_lst, Signed) if checkidx: - spec = dum_checkidx + hop.exception_is_here() else: - spec = dum_nocheck - v_func = hop.inputconst(Void, spec) - v_lst, v_index = hop.inputargs(r_lst, Signed) + hop.exception_cannot_occur() if hop.args_s[0].listdef.listitem.mutated or checkidx: if hop.args_s[1].nonneg: llfn = ll_getitem_nonneg else: llfn = ll_getitem + if checkidx: + spec = dum_checkidx + else: + spec = dum_nocheck + c_func_marker = hop.inputconst(Void, spec) + v_res = hop.gendirectcall(llfn, c_func_marker, v_lst, v_index) else: # this is the 'foldable' version, which is not used when # we check for IndexError @@ -259,11 +264,7 @@ llfn = ll_getitem_foldable_nonneg else: llfn = ll_getitem_foldable - if checkidx: - hop.exception_is_here() - else: - hop.exception_cannot_occur() - v_res = hop.gendirectcall(llfn, v_func, v_lst, v_index) + v_res = hop.gendirectcall(llfn, v_lst, v_index) return r_lst.recast(hop.llops, v_res) rtype_getitem_key = rtype_getitem @@ -538,12 +539,14 @@ dest.ll_setitem_fast(dest_start + i, item) i += 1 ll_arraycopy._annenforceargs_ = [None, None, int, int, int] +# no oopspec -- the function is inlined by the JIT def ll_copy(RESLIST, l): length = l.ll_length() new_lst = RESLIST.ll_newlist(length) ll_arraycopy(l, new_lst, 0, 0, length) return new_lst +# no oopspec -- the function is inlined by the JIT def ll_len(l): return l.ll_length() @@ -551,6 +554,7 @@ def ll_list_is_true(l): # check if a list is True, allowing for None return bool(l) and l.ll_length() != 0 +# no oopspec -- the function is inlined by the JIT def ll_len_foldable(l): return l.ll_length() @@ -558,6 +562,7 @@ def ll_list_is_true_foldable(l): return bool(l) and ll_len_foldable(l) != 0 +# no oopspec -- the function is inlined by the JIT def ll_append(l, newitem): length = l.ll_length() @@ -588,6 +593,7 @@ ll_arraycopy(l1, l, 0, 0, len1) ll_arraycopy(l2, l, 0, len1, len2) return l +# no oopspec -- the function is inlined by the JIT def ll_insert_nonneg(l, index, newitem): length = l.ll_length() @@ -674,60 +680,72 @@ l.ll_setitem_fast(length_1_i, tmp) i += 1 length_1_i -= 1 +ll_reverse.oopspec = 'list.reverse(l)' def ll_getitem_nonneg(func, l, index): ll_assert(index >= 0, "unexpectedly negative list getitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError - else: - ll_assert(index < l.ll_length(), "list getitem index out of bound") return l.ll_getitem_fast(index) -ll_getitem_nonneg.oopspec = 'list.getitem(l, index)' +ll_getitem_nonneg._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_getitem(func, l, index): - length = l.ll_length() - if index < 0: - index += length if func is dum_checkidx: - if index < 0 or index >= length: - raise IndexError + length = l.ll_length() # common case: 0 <= index < length + if r_uint(index) >= r_uint(length): + # Failed, so either (-length <= index < 0), or we have to raise + # IndexError. First add 'length' to get the final index, then + # check that we now have (0 <= index < length). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(index >= 0, "negative list getitem index out of bound") - ll_assert(index < length, "list getitem index out of bound") + # We don't want checking, but still want to support index < 0. + # Only call ll_length() if needed. + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list getitem index out of bound") return l.ll_getitem_fast(index) -ll_getitem.oopspec = 'list.getitem(l, index)' +# no oopspec -- the function is inlined by the JIT -def ll_getitem_foldable_nonneg(func, l, index): - return ll_getitem_nonneg(func, l, index) +def ll_getitem_foldable_nonneg(l, index): + ll_assert(index >= 0, "unexpectedly negative list getitem index") + return l.ll_getitem_fast(index) ll_getitem_foldable_nonneg.oopspec = 'list.getitem_foldable(l, index)' -def ll_getitem_foldable(func, l, index): - return ll_getitem(func, l, index) -ll_getitem_foldable.oopspec = 'list.getitem_foldable(l, index)' +def ll_getitem_foldable(l, index): + if index < 0: + index += l.ll_length() + return ll_getitem_foldable_nonneg(l, index) +ll_getitem_foldable._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_setitem_nonneg(func, l, index, newitem): ll_assert(index >= 0, "unexpectedly negative list setitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError - else: - ll_assert(index < l.ll_length(), "list setitem index out of bound") l.ll_setitem_fast(index, newitem) -ll_setitem_nonneg.oopspec = 'list.setitem(l, index, newitem)' +ll_setitem_nonneg._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_setitem(func, l, index, newitem): - length = l.ll_length() - if index < 0: - index += length if func is dum_checkidx: - if index < 0 or index >= length: - raise IndexError + length = l.ll_length() + if r_uint(index) >= r_uint(length): # see comments in ll_getitem(). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(index >= 0, "negative list setitem index out of bound") - ll_assert(index < length, "list setitem index out of bound") + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list setitem index out of bound") l.ll_setitem_fast(index, newitem) -ll_setitem.oopspec = 'list.setitem(l, index, newitem)' +# no oopspec -- the function is inlined by the JIT def ll_delitem_nonneg(func, l, index): ll_assert(index >= 0, "unexpectedly negative list delitem index") @@ -751,19 +769,20 @@ l._ll_resize_le(newlength) ll_delitem_nonneg.oopspec = 'list.delitem(l, index)' -def ll_delitem(func, l, i): - length = l.ll_length() - if i < 0: - i += length +def ll_delitem(func, l, index): if func is dum_checkidx: - if i < 0 or i >= length: - raise IndexError + length = l.ll_length() + if r_uint(index) >= r_uint(length): # see comments in ll_getitem(). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(i >= 0, "negative list delitem index out of bound") - ll_assert(i < length, "list delitem index out of bound") - ll_delitem_nonneg(dum_nocheck, l, i) -ll_delitem.oopspec = 'list.delitem(l, i)' - + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list delitem index out of bound") + ll_delitem_nonneg(dum_nocheck, l, index) +# no oopspec -- the function is inlined by the JIT def ll_extend(l1, l2): len1 = l1.ll_length() @@ -799,6 +818,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_str_slice_startstop(lst, s, getstrlen, getstritem, start, stop): @@ -824,6 +844,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_str_slice_minusone(lst, s, getstrlen, getstritem): len1 = lst.ll_length() @@ -843,6 +864,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_char_count(lst, char, count): if count <= 0: @@ -859,6 +881,7 @@ while j < newlength: lst.ll_setitem_fast(j, char) j += 1 +# not inlined by the JIT -- contains a loop def ll_listslice_startonly(RESLIST, l1, start): len1 = l1.ll_length() @@ -869,6 +892,7 @@ ll_arraycopy(l1, l, start, 0, newlength) return l ll_listslice_startonly._annenforceargs_ = (None, None, int) +# no oopspec -- the function is inlined by the JIT def ll_listslice_startstop(RESLIST, l1, start, stop): length = l1.ll_length() @@ -881,6 +905,7 @@ l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, start, 0, newlength) return l +# no oopspec -- the function is inlined by the JIT def ll_listslice_minusone(RESLIST, l1): newlength = l1.ll_length() - 1 @@ -888,6 +913,7 @@ l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, 0, 0, newlength) return l +# no oopspec -- the function is inlined by the JIT def ll_listdelslice_startonly(l, start): ll_assert(start >= 0, "del l[start:] with unexpectedly negative start") @@ -958,6 +984,7 @@ return False j += 1 return True +# not inlined by the JIT -- contains a loop def ll_listcontains(lst, obj, eqfn): lng = lst.ll_length() @@ -971,6 +998,7 @@ return True j += 1 return False +# not inlined by the JIT -- contains a loop def ll_listindex(lst, obj, eqfn): lng = lst.ll_length() @@ -984,6 +1012,7 @@ return j j += 1 raise ValueError # can't say 'list.index(x): x not in list' +# not inlined by the JIT -- contains a loop def ll_listremove(lst, obj, eqfn): index = ll_listindex(lst, obj, eqfn) # raises ValueError if obj not in lst @@ -1030,3 +1059,4 @@ i += 1 j += length return res +# not inlined by the JIT -- contains a loop Modified: pypy/branch/jitypes2/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/rpbc.py (original) +++ pypy/branch/jitypes2/pypy/rpython/rpbc.py Fri Dec 3 11:04:29 2010 @@ -15,11 +15,10 @@ from pypy.rpython import callparse - def small_cand(rtyper, s_pbc): if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets and \ hasattr(rtyper.type_system.rpbc, 'SmallFunctionSetPBCRepr'): - callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(rtyper, callfamily) if len(uniquerows) == 1 and (not s_pbc.subset_of or small_cand(rtyper, s_pbc.subset_of)): return True @@ -31,7 +30,7 @@ return none_frozen_pbc_repr kind = self.getKind() if issubclass(kind, description.FunctionDesc): - sample = self.descriptions.keys()[0] + sample = self.any_description() callfamily = sample.querycallfamily() if callfamily and callfamily.total_calltable_size > 0: if sample.overridden: @@ -181,7 +180,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None: # a single function self.lowleveltype = Void @@ -207,7 +206,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) ## def function_signatures(self): @@ -322,7 +321,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness @@ -368,7 +367,7 @@ return robject.pyobj_repr def getFrozenPBCRepr(rtyper, s_pbc): - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) assert len(descs) >= 1 if len(descs) == 1 and not s_pbc.can_be_None: return SingleFrozenPBCRepr(descs[0]) @@ -530,7 +529,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper - self.funcdesc = s_pbc.descriptions.keys()[0].funcdesc + self.funcdesc = s_pbc.any_description().funcdesc # a hack to force the underlying function to show up in call_families # (generally not needed, as normalizecalls() should ensure this, @@ -662,7 +661,7 @@ and the ClassRepr of the class which stores this attribute in its vtable. """ - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) access = classdescs[0].queryattrfamily(attrname) for classdesc in classdescs[1:]: access1 = classdesc.queryattrfamily(attrname) @@ -819,7 +818,7 @@ if s_pbc.isNone(): raise TyperError("unsupported: variable of type " "bound-method-object or None") - mdescs = s_pbc.descriptions.keys() + mdescs = list(s_pbc.descriptions) methodname = mdescs[0].name classdef = mdescs[0].selfclassdef flags = mdescs[0].flags Modified: pypy/branch/jitypes2/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/jitypes2/pypy/rpython/test/test_rclass.py Fri Dec 3 11:04:29 2010 @@ -917,6 +917,7 @@ assert destrptr is not None def test_del_inheritance(self): + from pypy.rlib import rgc class State: pass s = State() @@ -937,6 +938,7 @@ A() B() C() + rgc.collect() return s.a_dels * 10 + s.b_dels res = f() assert res == 42 @@ -1067,6 +1069,7 @@ assert meth.finalizer def test_del_inheritance(self): + from pypy.rlib import rgc class State: pass s = State() @@ -1087,6 +1090,7 @@ A() B() C() + rgc.collect() return s.a_dels * 10 + s.b_dels res = f() assert res == 42 Modified: pypy/branch/jitypes2/pypy/rpython/test/test_rint.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/test/test_rint.py (original) +++ pypy/branch/jitypes2/pypy/rpython/test/test_rint.py Fri Dec 3 11:04:29 2010 @@ -4,14 +4,9 @@ from pypy.annotation import model as annmodel from pypy.rpython.test import snippet from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong -from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.rarithmetic import ovfcheck, r_int64 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -if r_longlong is not r_int: - int64 = r_longlong -else: - int64 = int - class TestSnippet(object): @@ -110,10 +105,10 @@ def f(i): return str(i) - res = self.interpret(f, [int64(0)]) + res = self.interpret(f, [r_int64(0)]) assert self.ll_to_string(res) == '0' - res = self.interpret(f, [int64(413974738222117)]) + res = self.interpret(f, [r_int64(413974738222117)]) assert self.ll_to_string(res) == '413974738222117' def test_unsigned(self): @@ -135,7 +130,7 @@ f._annspecialcase_ = "specialize:argtype(0)" def g(n): if n > 0: - return f(int64(0)) + return f(r_int64(0)) else: return f(0) res = self.interpret(g, [0]) @@ -147,7 +142,7 @@ def test_downcast_int(self): def f(i): return int(i) - res = self.interpret(f, [int64(0)]) + res = self.interpret(f, [r_int64(0)]) assert res == 0 def test_isinstance_vs_int_types(self): @@ -157,7 +152,7 @@ return [None] if isinstance(x, str): return x - if isinstance(x, int64): + if isinstance(x, r_int64): return int(x) return "XXX" wrap._annspecialcase_ = 'specialize:argtype(0)' @@ -165,7 +160,7 @@ space = FakeSpace() def wrap(x): return space.wrap(x) - res = self.interpret(wrap, [int64(0)]) + res = self.interpret(wrap, [r_int64(0)]) assert res == 0 def test_truediv(self): @@ -178,25 +173,25 @@ def test_float_conversion(self): def f(ii): return float(ii) - res = self.interpret(f, [int64(100000000)]) + res = self.interpret(f, [r_int64(100000000)]) assert type(res) is float assert res == 100000000. - res = self.interpret(f, [int64(1234567890123456789)]) + res = self.interpret(f, [r_int64(1234567890123456789)]) assert type(res) is float assert self.float_eq(res, 1.2345678901234568e+18) def test_float_conversion_implicit(self): def f(ii): return 1.0 + ii - res = self.interpret(f, [int64(100000000)]) + res = self.interpret(f, [r_int64(100000000)]) assert type(res) is float assert res == 100000001. - res = self.interpret(f, [int64(1234567890123456789)]) + res = self.interpret(f, [r_int64(1234567890123456789)]) assert type(res) is float assert self.float_eq(res, 1.2345678901234568e+18) def test_rarithmetic(self): - inttypes = [int, r_uint, int64, r_ulonglong] + inttypes = [int, r_uint, r_int64, r_ulonglong] for inttype in inttypes: c = inttype() def f(): @@ -231,16 +226,16 @@ res = self.interpret(f, [int(-1<<(r_int.BITS-1))]) assert res == 0 - res = self.interpret(f, [int64(-1)]) + res = self.interpret(f, [r_int64(-1)]) assert res == 1 - res = self.interpret(f, [int64(-1)<<(r_longlong.BITS-1)]) + res = self.interpret(f, [r_int64(-1)<<(r_longlong.BITS-1)]) assert res == 0 div_mod_iteration_count = 1000 def test_div_mod(self): import random - for inttype in (int, int64): + for inttype in (int, r_int64): def d(x, y): return x/y @@ -303,7 +298,7 @@ except ZeroDivisionError: return 84 - for inttype in (int, int64): + for inttype in (int, r_int64): args = [( 5, 2), (-5, 2), ( 5,-2), (-5,-2), ( 6, 2), (-6, 2), ( 6,-2), (-6,-2), Modified: pypy/branch/jitypes2/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/test/test_rlist.py (original) +++ pypy/branch/jitypes2/pypy/rpython/test/test_rlist.py Fri Dec 3 11:04:29 2010 @@ -12,6 +12,7 @@ from pypy.rpython.rint import signed_repr from pypy.objspace.flow.model import Constant, Variable from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +from pypy.rlib.debug import ll_assert # undo the specialization parameter for n1 in 'get set del'.split(): @@ -1076,7 +1077,13 @@ res = self.interpret(f, [0]) assert res == 1 - py.test.raises(AssertionError, self.interpret, f, [1]) + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) + else: + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def f(x): l = [1] @@ -1121,12 +1128,13 @@ res = self.interpret(f, [0]) assert res == 1 - try: - self.interpret_raises(IndexError, f, [1]) - except (AssertionError,), e: - pass + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) else: - assert False + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def f(x): l = [1] @@ -1163,12 +1171,13 @@ res = self.interpret(f, [0]) assert res == 1 - try: - self.interpret_raises(IndexError, f, [1]) - except (AssertionError,), e: - pass + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) else: - assert False + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def test_charlist_extension_1(self): def f(n): @@ -1327,8 +1336,32 @@ res = self.interpret(f, [2]) assert res == True + def test_immutable_list_out_of_instance(self): + from pypy.translator.simplify import get_funcobj + for immutable_fields in (["a", "b"], ["a", "b", "y[*]"]): + class A(object): + _immutable_fields_ = immutable_fields + class B(A): + pass + def f(i): + b = B() + lst = [i] + lst[0] += 1 + b.y = lst + ll_assert(b.y is lst, "copying when reading out the attr?") + return b.y[0] + res = self.interpret(f, [10]) + assert res == 11 + t, rtyper, graph = self.gengraph(f, [int]) + block = graph.startblock + op = block.operations[-1] + assert op.opname == 'direct_call' + func = get_funcobj(op.args[0].value)._callable + assert ('foldable' in func.func_name) == \ + ("y[*]" in immutable_fields) class TestLLtype(BaseTestRlist, LLRtypeMixin): + type_system = 'lltype' rlist = ll_rlist def test_memoryerror(self): @@ -1420,14 +1453,16 @@ lst2 = [i] lst2.append(42) # mutated list return lst1[i] + lst2[i] - _, _, graph = self.gengraph(f, [int]) + from pypy.annotation import model as annmodel + _, _, graph = self.gengraph(f, [annmodel.SomeInteger(nonneg=True)]) block = graph.startblock lst1_getitem_op = block.operations[-3] # XXX graph fishing lst2_getitem_op = block.operations[-2] func1 = lst1_getitem_op.args[0].value._obj._callable func2 = lst2_getitem_op.args[0].value._obj._callable assert func1.oopspec == 'list.getitem_foldable(l, index)' - assert func2.oopspec == 'list.getitem(l, index)' + assert not hasattr(func2, 'oopspec') class TestOOtype(BaseTestRlist, OORtypeMixin): rlist = oo_rlist + type_system = 'ootype' Modified: pypy/branch/jitypes2/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/jitypes2/pypy/rpython/tool/rffi_platform.py Fri Dec 3 11:04:29 2010 @@ -639,6 +639,7 @@ C_HEADER = """ #include #include /* for offsetof() */ +#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); Modified: pypy/branch/jitypes2/pypy/tool/ansi_print.py ============================================================================== --- pypy/branch/jitypes2/pypy/tool/ansi_print.py (original) +++ pypy/branch/jitypes2/pypy/tool/ansi_print.py Fri Dec 3 11:04:29 2010 @@ -16,6 +16,7 @@ 'WARNING': ((31,), False), 'event': ((1,), True), 'ERROR': ((1, 31), False), + 'Error': ((1, 31), False), 'info': ((35,), False), 'stub': ((34,), False), } Modified: pypy/branch/jitypes2/pypy/tool/logparser.py ============================================================================== --- pypy/branch/jitypes2/pypy/tool/logparser.py (original) +++ pypy/branch/jitypes2/pypy/tool/logparser.py Fri Dec 3 11:04:29 2010 @@ -12,14 +12,6 @@ from pypy.tool import progressbar def parse_log_file(filename, verbose=True): - r_start = re.compile(r"\[([0-9a-fA-F]+)\] \{([\w-]+)$") - r_stop = re.compile(r"\[([0-9a-fA-F]+)\] ([\w-]+)\}$") - lasttime = 0 - log = DebugLog() - time_decrase = False - performance_log = True - nested = 0 - # f = open(filename, 'r') if f.read(2) == 'BZ': f.close() @@ -30,19 +22,33 @@ lines = f.readlines() f.close() # - if sys.stdout.isatty(): + return parse_log(lines, verbose=verbose) + +def parse_log(lines, verbose=False): + color = "(?:\x1b.*?m)?" + r_start = re.compile(color + r"\[([0-9a-fA-F]+)\] \{([\w-]+)" + color + "$") + r_stop = re.compile(color + r"\[([0-9a-fA-F]+)\] ([\w-]+)\}" + color + "$") + lasttime = 0 + log = DebugLog() + time_decrase = False + performance_log = True + nested = 0 + # + if verbose and sys.stdout.isatty(): progress = progressbar.ProgressBar(color='green') + counter = 0 + else: + progress = None single_percent = len(lines) / 100 if verbose: - vnext = single_percent + vnext = 0 else: - vnext = len(lines) - counter = 0 + vnext = -1 for i, line in enumerate(lines): if i == vnext: - counter += 1 - if sys.stdout.isatty(): + if progress is not None: progress.render(counter) + counter += 1 vnext += single_percent else: sys.stderr.write('%d%%..' % int(100.0*i/len(lines))) Modified: pypy/branch/jitypes2/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/c/funcgen.py (original) +++ pypy/branch/jitypes2/pypy/translator/c/funcgen.py Fri Dec 3 11:04:29 2010 @@ -1,3 +1,4 @@ +import sys from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring from pypy.translator.c.support import cdecl from pypy.translator.c.support import llvalue_from_constant, gen_assignments @@ -757,6 +758,16 @@ format.append('%s') argv.append('(%s) ? "True" : "False"' % self.expr(arg)) continue + elif T == SignedLongLong: + if sys.platform == 'win32': + format.append('%I64d') + else: + format.append('%lld') + elif T == UnsignedLongLong: + if sys.platform == 'win32': + format.append('%I64u') + else: + format.append('%llu') else: raise Exception("don't know how to debug_print %r" % (T,)) argv.append(self.expr(arg)) @@ -765,17 +776,20 @@ "if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}" % (', '.join(argv), free_line)) + def _op_debug(self, opname, arg): + if isinstance(arg, Constant): + string_literal = c_string_constant(''.join(arg.value.chars)) + return "%s(%s);" % (opname, string_literal) + else: + x = "%s(RPyString_AsCharP(%s));\n" % (opname, self.expr(arg)) + x += "RPyString_FreeCache();" + return x + def OP_DEBUG_START(self, op): - arg = op.args[0] - assert isinstance(arg, Constant) - return "PYPY_DEBUG_START(%s);" % ( - c_string_constant(''.join(arg.value.chars)),) + return self._op_debug('PYPY_DEBUG_START', op.args[0]) def OP_DEBUG_STOP(self, op): - arg = op.args[0] - assert isinstance(arg, Constant) - return "PYPY_DEBUG_STOP(%s);" % ( - c_string_constant(''.join(arg.value.chars)),) + return self._op_debug('PYPY_DEBUG_STOP', op.args[0]) def OP_DEBUG_ASSERT(self, op): return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]), Modified: pypy/branch/jitypes2/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/jitypes2/pypy/translator/c/gcc/trackgcroot.py Fri Dec 3 11:04:29 2010 @@ -1106,7 +1106,7 @@ format = 'darwin64' function_names_prefix = '_' - LABEL = ElfFunctionGcRootTracker32.LABEL + LABEL = ElfFunctionGcRootTracker64.LABEL r_jmptable_item = re.compile(r"\t.(?:long|quad)\t"+LABEL+"(-\"?[A-Za-z0-9$]+\"?)?\s*$") r_functionstart = re.compile(r"_(\w+):\s*$") Modified: pypy/branch/jitypes2/pypy/translator/c/src/debug_alloc.h ============================================================================== --- pypy/branch/jitypes2/pypy/translator/c/src/debug_alloc.h (original) +++ pypy/branch/jitypes2/pypy/translator/c/src/debug_alloc.h Fri Dec 3 11:04:29 2010 @@ -1,5 +1,5 @@ /**************************************************************/ - /*** tracking raw mallocs and frees for debugging ***/ +/*** tracking raw mallocs and frees for debugging ***/ #ifndef RPY_ASSERT @@ -62,8 +62,8 @@ count++; if (count > 0) { - fprintf(stderr, "debug_alloc.h: %ld mallocs left", count); char *env = getenv("PYPY_ALLOC"); + fprintf(stderr, "debug_alloc.h: %ld mallocs left", count); if (env && *env) { fprintf(stderr, " (most recent first):\n"); Modified: pypy/branch/jitypes2/pypy/translator/c/src/debug_print.h ============================================================================== --- pypy/branch/jitypes2/pypy/translator/c/src/debug_print.h (original) +++ pypy/branch/jitypes2/pypy/translator/c/src/debug_print.h Fri Dec 3 11:04:29 2010 @@ -72,7 +72,11 @@ { char *filename = getenv("PYPYLOG"); if (filename) +#ifndef MS_WINDOWS unsetenv("PYPYLOG"); /* don't pass it to subprocesses */ +#else + putenv("PYPYLOG="); /* don't pass it to subprocesses */ +#endif if (filename && filename[0]) { char *colon = strchr(filename, ':'); Modified: pypy/branch/jitypes2/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/jitypes2/pypy/translator/c/src/mem.h (original) +++ pypy/branch/jitypes2/pypy/translator/c/src/mem.h Fri Dec 3 11:04:29 2010 @@ -233,4 +233,4 @@ #define OP_GC_GET_RPY_MEMORY_USAGE(x, r) r = -1 #define OP_GC_GET_RPY_TYPE_INDEX(x, r) r = -1 #define OP_GC_IS_RPY_INSTANCE(x, r) r = 0 -#define OP_GC_DUMP_RPY_HEAP(r) r = 0 +#define OP_GC_DUMP_RPY_HEAP(fd, r) r = 0 Modified: pypy/branch/jitypes2/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/jitypes2/pypy/translator/c/test/test_newgc.py Fri Dec 3 11:04:29 2010 @@ -1064,14 +1064,16 @@ def test_get_rpy_type_index(self): self.run("get_rpy_type_index") - filename_dump = str(udir.join('test_dump_rpy_heap')) + filename1_dump = str(udir.join('test_dump_rpy_heap.1')) + filename2_dump = str(udir.join('test_dump_rpy_heap.2')) def define_dump_rpy_heap(self): U = lltype.GcForwardReference() U.become(lltype.GcStruct('U', ('next', lltype.Ptr(U)), ('x', lltype.Signed))) S = lltype.GcStruct('S', ('u', lltype.Ptr(U))) A = lltype.GcArray(lltype.Ptr(S)) - filename = self.filename_dump + filename1 = self.filename1_dump + filename2 = self.filename2_dump def fn(): s = lltype.malloc(S) @@ -1081,20 +1083,31 @@ a = lltype.malloc(A, 1000) s2 = lltype.malloc(S) # - fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0666) - rgc.dump_rpy_heap(fd) + fd1 = os.open(filename1, os.O_WRONLY | os.O_CREAT, 0666) + fd2 = os.open(filename2, os.O_WRONLY | os.O_CREAT, 0666) + rgc.dump_rpy_heap(fd1) + rgc.dump_rpy_heap(fd2) # try twice in a row keepalive_until_here(s2) keepalive_until_here(s) keepalive_until_here(a) - os.close(fd) + os.close(fd1) + os.close(fd2) return 0 return fn def test_dump_rpy_heap(self): self.run("dump_rpy_heap") - assert os.path.exists(self.filename_dump) - assert os.path.getsize(self.filename_dump) > 64 + for fn in [self.filename1_dump, self.filename2_dump]: + assert os.path.exists(fn) + assert os.path.getsize(fn) > 64 + f = open(self.filename1_dump) + data1 = f.read() + f.close() + f = open(self.filename2_dump) + data2 = f.read() + f.close() + assert data1 == data2 filename_dump_typeids_z = str(udir.join('test_typeids_z')) def define_write_typeids_z(self): Modified: pypy/branch/jitypes2/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/jitypes2/pypy/translator/c/test/test_standalone.py Fri Dec 3 11:04:29 2010 @@ -272,7 +272,7 @@ x = "got:" debug_start ("mycat") if have_debug_prints(): x += "b" - debug_print ("foo", 2, "bar", 3) + debug_print ("foo", r_longlong(2), "bar", 3) debug_start ("cat2") if have_debug_prints(): x += "c" debug_print ("baz") @@ -403,6 +403,20 @@ assert not err assert path.check(file=0) + def test_debug_print_start_stop_nonconst(self): + def entry_point(argv): + debug_start(argv[1]) + debug_print(argv[2]) + debug_stop(argv[1]) + return 0 + t, cbuilder = self.compile(entry_point) + out, err = cbuilder.cmdexec("foo bar", err=True, env={'PYPYLOG': ':-'}) + lines = err.splitlines() + assert '{foo' in lines[0] + assert 'bar' == lines[1] + assert 'foo}' in lines[2] + + def test_fatal_error(self): def g(x): if x == 1: Modified: pypy/branch/jitypes2/pypy/translator/driver.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/driver.py (original) +++ pypy/branch/jitypes2/pypy/translator/driver.py Fri Dec 3 11:04:29 2010 @@ -11,7 +11,7 @@ from pypy.annotation import policy as annpolicy import optparse from pypy.tool.udir import udir -from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS +from pypy.tool.debug_print import debug_start, debug_print, debug_stop from pypy.rlib.entrypoint import secondary_entrypoints import py @@ -37,13 +37,6 @@ 'c': 'lltype', } -JIT_DEBUG = { - 'off' : DEBUG_OFF, - 'profile' : DEBUG_PROFILE, - 'steps' : DEBUG_STEPS, - 'detailed' : DEBUG_DETAILED, -} - def backend_to_typesystem(backend): return _BACKEND_TO_TYPESYSTEM.get(backend, 'ootype') @@ -283,6 +276,8 @@ return else: self.log.info("%s..." % title) + debug_start('translation-task') + debug_print('starting', goal) self.timer.start_event(goal) try: instrument = False @@ -300,11 +295,13 @@ assert False, 'we should not get here' finally: try: + debug_stop('translation-task') self.timer.end_event(goal) except (KeyboardInterrupt, SystemExit): raise except: pass + #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal) return res def task_annotate(self): @@ -399,7 +396,6 @@ # from pypy.jit.metainterp.warmspot import apply_jit apply_jit(self.translator, policy=self.jitpolicy, - debug_level=JIT_DEBUG[self.config.translation.jit_debug], backend_name=self.config.translation.jit_backend, inline=True) # self.log.info("the JIT compiler was generated") @@ -417,7 +413,6 @@ # from pypy.jit.metainterp.warmspot import apply_jit apply_jit(self.translator, policy=self.jitpolicy, - debug_level=JIT_DEBUG[self.config.translation.jit_debug], backend_name='cli', inline=True) #XXX # self.log.info("the JIT compiler was generated") Modified: pypy/branch/jitypes2/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/goal/app_main.py (original) +++ pypy/branch/jitypes2/pypy/translator/goal/app_main.py Fri Dec 3 11:04:29 2010 @@ -11,6 +11,7 @@ -h, --help show this help message and exit -m library module to be run as a script (terminates option list) -W arg warning control (arg is action:message:category:module:lineno) + -E ignore environment variables (such as PYTHONPATH) --version print the PyPy version --info print translation information about this PyPy executable """ @@ -199,7 +200,7 @@ break # found! return newpath -def setup_initial_paths(executable, nanos): +def setup_initial_paths(executable, nanos, readenv=True, **extra): # a substituted os if we are translated global os os = nanos @@ -220,7 +221,7 @@ sys.executable = os.path.abspath(executable) newpath = get_library_path(executable) - path = os.getenv('PYTHONPATH') + path = readenv and os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath # remove duplicates @@ -230,7 +231,6 @@ if dir not in _seen: sys.path.append(dir) _seen[dir] = True - return executable def parse_command_line(argv): @@ -242,6 +242,7 @@ run_stdin = False warnoptions = [] unbuffered = False + readenv = True while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -253,6 +254,8 @@ argv[i] = '-c' run_command = True break + elif arg == '-E': + readenv = False elif arg == '-u': unbuffered = True elif arg == '-O' or arg == '-OO': @@ -305,6 +308,7 @@ run_stdin, warnoptions, unbuffered, + readenv, cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 @@ -355,7 +359,7 @@ # * PYTHONINSPECT is set and stdin is a tty. # return (go_interactive or - (os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) + (readenv and os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) success = True @@ -388,7 +392,7 @@ # If stdin is a tty or if "-i" is specified, we print # a banner and run $PYTHONSTARTUP. print_banner() - python_startup = os.getenv('PYTHONSTARTUP') + python_startup = readenv and os.getenv('PYTHONSTARTUP') if python_startup: try: f = open(python_startup) @@ -451,7 +455,6 @@ '"license" for more information.') def entry_point(executable, argv, nanos): - executable = setup_initial_paths(executable, nanos) try: cmdline = parse_command_line(argv) except CommandLineError, e: @@ -459,8 +462,8 @@ return 2 if cmdline is None: return 0 - else: - return run_command_line(**cmdline) + setup_initial_paths(executable, nanos, **cmdline) + return run_command_line(**cmdline) if __name__ == '__main__': @@ -495,13 +498,13 @@ sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path os = nanos.os_module_for_testing - sys.ps1 = '>>>> ' - sys.ps2 = '.... ' try: sys.exit(int(entry_point(sys.argv[0], sys.argv[1:], os))) finally: - sys.ps1 = '>>> ' # restore the normal ones, in case - sys.ps2 = '... ' # we are dropping to CPython's prompt + # restore the normal prompt (which was changed by _pypy_interact), in + # case we are dropping to CPython's prompt + sys.ps1 = '>>> ' + sys.ps2 = '... ' import os; os.environ.update(reset) assert old_argv is sys.argv assert old_path is sys.path Modified: pypy/branch/jitypes2/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/jitypes2/pypy/translator/goal/test2/test_app_main.py Fri Dec 3 11:04:29 2010 @@ -217,6 +217,38 @@ finally: os.environ['PYTHONSTARTUP'] = old + def test_ignore_python_startup(self): + old = os.environ.get('PYTHONSTARTUP', '') + try: + os.environ['PYTHONSTARTUP'] = crashing_demo_script + child = self.spawn(['-E']) + child.expect(re.escape(banner)) + index = child.expect(['Traceback', '>>> ']) + assert index == 1 # no traceback + finally: + os.environ['PYTHONSTARTUP'] = old + + def test_ignore_python_inspect(self): + os.environ['PYTHONINSPECT_'] = '1' + try: + child = self.spawn(['-E', '-c', 'pass']) + from pexpect import EOF + index = child.expect(['>>> ', EOF]) + assert index == 1 # no prompt + finally: + del os.environ['PYTHONINSPECT_'] + + def test_ignore_python_path(self): + old = os.environ.get('PYTHONPATH', '') + try: + os.environ['PYTHONPATH'] = 'foobarbaz' + child = self.spawn(['-E', '-c', 'import sys; print sys.path']) + from pexpect import EOF + index = child.expect(['foobarbaz', EOF]) + assert index == 1 # no foobarbaz + finally: + os.environ['PYTHONPATH'] = old + def test_unbuffered(self): line = 'import os,sys;sys.stdout.write(str(789));os.read(0,1)' child = self.spawn(['-u', '-c', line]) @@ -329,6 +361,10 @@ child = self.spawn(['-mpypy.translator.goal.test2.mymodule']) child.expect('mymodule running') + def test_ps1_only_if_interactive(self): + argv = ['-c', 'import sys; print hasattr(sys, "ps1")'] + child = self.spawn(argv) + child.expect('False') class TestNonInteractive: Modified: pypy/branch/jitypes2/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/platform/__init__.py (original) +++ pypy/branch/jitypes2/pypy/translator/platform/__init__.py Fri Dec 3 11:04:29 2010 @@ -123,7 +123,9 @@ errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() for line in stderrlines: - log.ERROR(line) + log.Error(line) + # ^^^ don't use ERROR, because it might actually be fine. + # Also, ERROR confuses lib-python/conftest.py. raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): @@ -215,13 +217,13 @@ host_factory = Darwin_i386 else: host_factory = Darwin_x86_64 -elif sys.platform == 'freebsd7': - from pypy.translator.platform.freebsd7 import Freebsd7, Freebsd7_64 +elif "freebsd" in sys.platform: + from pypy.translator.platform.freebsd import Freebsd, Freebsd_64 import platform if platform.architecture()[0] == '32bit': - host_factory = Freebsd7 + host_factory = Freebsd else: - host_factory = Freebsd7_64 + host_factory = Freebsd_64 elif os.name == 'nt': from pypy.translator.platform.windows import Windows host_factory = Windows Modified: pypy/branch/jitypes2/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/platform/darwin.py (original) +++ pypy/branch/jitypes2/pypy/translator/platform/darwin.py Fri Dec 3 11:04:29 2010 @@ -12,12 +12,14 @@ so_ext = 'so' + default_cc = 'gcc' + def __init__(self, cc=None): if cc is None: try: cc = os.environ['CC'] except KeyError: - cc = 'gcc' + cc = self.default_cc self.cc = cc def _args_for_shared(self, args): @@ -85,3 +87,4 @@ link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4') cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4') + default_cc = 'gcc-4.0' Modified: pypy/branch/jitypes2/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/jitypes2/pypy/translator/tool/cbuild.py Fri Dec 3 11:04:29 2010 @@ -309,6 +309,7 @@ #define _POSIX_C_SOURCE 200112L /* Define on FreeBSD to activate all library features */ #define __BSD_VISIBLE 1 +#define __XSI_VISIBLE 700 /* Windows: winsock/winsock2 mess */ #define WIN32_LEAN_AND_MEAN ''' Modified: pypy/branch/jitypes2/pypy/translator/tool/reftracker.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/tool/reftracker.py (original) +++ pypy/branch/jitypes2/pypy/translator/tool/reftracker.py Fri Dec 3 11:04:29 2010 @@ -3,7 +3,7 @@ Usage: call track(obj). """ -import autopath, sys, os +import autopath, sys, os, types import gc from pypy.translator.tool.graphpage import GraphPage, DotGen from pypy.tool.uid import uid @@ -39,7 +39,7 @@ if o2 is None: continue addedge(objectlist[i], o2) - id2typename[uid(o2)] = type(o2).__name__ + id2typename[uid(o2)] = self.shortrepr(o2) del o2 for o2 in self.get_referrers(objectlist[i]): if o2 is None: @@ -47,7 +47,7 @@ if type(o2) is list and o2 and o2[0] is MARKER: continue addedge(o2, objectlist[i]) - id2typename[uid(o2)] = type(o2).__name__ + id2typename[uid(o2)] = self.shortrepr(o2) del o2 for ids, label in edges.items(): @@ -82,13 +82,23 @@ return self.newpage(objectlist) def formatobject(self, o): + header = self.shortrepr(o, compact=False) + secondline = repr(o.__class__) + return header, secondline, repr(o) + + def shortrepr(self, o, compact=True): + t = type(o) + if t is types.FrameType: + if compact: + return 'frame %r' % (o.f_code.co_name,) + else: + return 'frame %r' % (o.f_code,) s = repr(o) if len(s) > 50: - linktext = s s = s[:20] + ' ... ' + s[-20:] - else: - linktext = '' - return type(o).__name__, s, linktext + if s.startswith('<') and s.endswith('>'): + s = s[1:-1] + return s def edgelabel(self, o1, o2): return '' From antocuni at codespeak.net Fri Dec 3 11:41:01 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Dec 2010 11:41:01 +0100 (CET) Subject: [pypy-svn] r79771 - pypy/branch/jitypes2/lib_pypy/_ctypes Message-ID: <20101203104101.A8433282BE0@codespeak.net> Author: antocuni Date: Fri Dec 3 11:40:58 2010 New Revision: 79771 Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Log: do the char-->int conversion based on the declared type, not on the actual value of the argument Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Fri Dec 3 11:40:58 2010 @@ -385,8 +385,9 @@ newargs = [] for argtype, arg in zip(argtypes, args): value = arg.value - if isinstance(value, basestring) and len(value) == 1: + if argtype._ffishape == 'u': # XXX: who should do this conversion? Maybe _ffi? + assert isinstance(value, basestring) and len(value) == 1 value = ord(value) newargs.append(value) return newargs From antocuni at codespeak.net Fri Dec 3 11:47:54 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Dec 2010 11:47:54 +0100 (CET) Subject: [pypy-svn] r79772 - pypy/branch/jitypes2/lib_pypy/_ctypes Message-ID: <20101203104754.6737B282BE3@codespeak.net> Author: antocuni Date: Fri Dec 3 11:47:45 2010 New Revision: 79772 Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Log: make test_wchar_result passing. Probably the details of type conversion will be refactored later Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Fri Dec 3 11:47:45 2010 @@ -199,6 +199,9 @@ restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype, thisarg) result = funcptr(*newargs) + if restype._ffishape == 'u': + # XXX: maybe it's a job of _ffi? + result = unichr(result) ## resbuffer = funcptr(*[arg._get_buffer_for_param()._buffer ## for arg in args]) ## result = self._build_result(restype, resbuffer, argtypes, args) From arigo at codespeak.net Fri Dec 3 13:03:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 3 Dec 2010 13:03:43 +0100 (CET) Subject: [pypy-svn] r79773 - pypy/trunk/pypy/annotation Message-ID: <20101203120343.9DE04282BD4@codespeak.net> Author: arigo Date: Fri Dec 3 13:03:39 2010 New Revision: 79773 Modified: pypy/trunk/pypy/annotation/binaryop.py Log: Fix to get a meaningful error message instead of TypeError. Modified: pypy/trunk/pypy/annotation/binaryop.py ============================================================================== --- pypy/trunk/pypy/annotation/binaryop.py (original) +++ pypy/trunk/pypy/annotation/binaryop.py Fri Dec 3 13:03:39 2010 @@ -861,7 +861,7 @@ def getitem((p, obj)): assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - def setitem((p, obj)): + def setitem((p, obj), s_value): assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) class __extend__(pairtype(SomeObject, SomePtr)): From antocuni at codespeak.net Fri Dec 3 13:28:16 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Dec 2010 13:28:16 +0100 (CET) Subject: [pypy-svn] r79774 - in pypy/branch/jitypes2/pypy/module/_ffi: . test Message-ID: <20101203122816.5B6CA282BE3@codespeak.net> Author: antocuni Date: Fri Dec 3 13:28:14 2010 New Revision: 79774 Modified: pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py Log: add support for pointers as arguments/return value. Since _ffi is only a thin layer, pointers are represented just as integers at app-level Modified: pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py Fri Dec 3 13:28:14 2010 @@ -124,6 +124,8 @@ intres = rffi.cast(rffi.LONG, call(argchain, rffi.SHORT)) elif restype is libffi.types.schar: intres = rffi.cast(rffi.LONG, call(argchain, rffi.SIGNEDCHAR)) + elif restype is libffi.types.pointer: + intres = rffi.cast(rffi.LONG, call(argchain, rffi.VOIDP)) else: raise OperationError(space.w_ValueError, space.wrap('Unsupported restype')) Modified: pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py Fri Dec 3 13:28:14 2010 @@ -103,6 +103,26 @@ assert get_dummy() == 0 assert set_dummy(42) is None assert get_dummy() == 42 + set_dummy(0) + + def test_pointer_args(self): + """ + extern int dummy; // defined in test_void_result + int* get_dummy_ptr() { return &dummy; } + void set_val_to_ptr(int* ptr, int val) { *ptr = val; } + """ + from _ffi import CDLL, types + libfoo = CDLL(self.libfoo_name) + get_dummy = libfoo.getfunc('get_dummy', [], types.sint) + get_dummy_ptr = libfoo.getfunc('get_dummy_ptr', [], types.pointer) + set_val_to_ptr = libfoo.getfunc('set_val_to_ptr', + [types.pointer, types.sint], + types.void) + assert get_dummy() == 0 + ptr = get_dummy_ptr() + set_val_to_ptr(ptr, 123) + assert get_dummy() == 123 + set_val_to_ptr(ptr, 0) def test_unsigned_long_args(self): """ @@ -147,6 +167,7 @@ res = sum_xy(12.34, 56.78) assert res == self.f_12_34_plus_56_78 + def test_TypeError_numargs(self): from _ffi import CDLL, types libfoo = CDLL(self.libfoo_name) From fijal at codespeak.net Fri Dec 3 13:33:31 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Fri, 3 Dec 2010 13:33:31 +0100 (CET) Subject: [pypy-svn] r79775 - in pypy/branch/out-of-line-guards/pypy: interpreter module/pypyjit module/sys Message-ID: <20101203123331.CE5CE282BE7@codespeak.net> Author: fijal Date: Fri Dec 3 13:33:30 2010 New Revision: 79775 Modified: pypy/branch/out-of-line-guards/pypy/interpreter/function.py pypy/branch/out-of-line-guards/pypy/module/pypyjit/interp_jit.py pypy/branch/out-of-line-guards/pypy/module/sys/__init__.py pypy/branch/out-of-line-guards/pypy/module/sys/vm.py Log: A bit of improvements on calling by reducing number of getfields. To make it work completely have to finish out of line guards Modified: pypy/branch/out-of-line-guards/pypy/interpreter/function.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/function.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/function.py Fri Dec 3 13:33:30 2010 @@ -29,7 +29,9 @@ can_change_code = True _immutable_fields_ = ['w_func_globals', 'closure'] + _jit_invariant_fields_ = ['defs_w', 'code'] + @jit.dont_look_inside def __init__(self, space, code, w_globals=None, defs_w=[], closure=None, forcename=None): self.space = space @@ -334,6 +336,7 @@ return space.w_None return space.newtuple(values_w) + @jit.dont_look_inside def fset_func_defaults(space, self, w_defaults): if space.is_w(w_defaults, space.w_None): self.defs_w = [] @@ -342,6 +345,7 @@ raise OperationError( space.w_TypeError, space.wrap("func_defaults must be set to a tuple object or None") ) self.defs_w = space.fixedview(w_defaults) + @jit.dont_look_inside def fdel_func_defaults(space, self): self.defs_w = [] @@ -387,6 +391,7 @@ def fget_func_code(space, self): return space.wrap(self.code) + @jit.dont_look_inside def fset_func_code(space, self, w_code): from pypy.interpreter.pycode import PyCode if not self.can_change_code: Modified: pypy/branch/out-of-line-guards/pypy/module/pypyjit/interp_jit.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/pypyjit/interp_jit.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/pypyjit/interp_jit.py Fri Dec 3 13:33:30 2010 @@ -21,6 +21,8 @@ 'fastlocals_w[*]', 'last_exception', 'lastblock', + 'w_globals', + 'is_being_profiled', ] JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] Modified: pypy/branch/out-of-line-guards/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/sys/__init__.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/sys/__init__.py Fri Dec 3 13:33:30 2010 @@ -5,6 +5,9 @@ class Module(MixedModule): """Sys Builtin Module. """ + + _jit_invariant_fields_ = ['recursionlimit'] + def __init__(self, space, w_name): """NOT_RPYTHON""" # because parent __init__ isn't if space.config.translating: Modified: pypy/branch/out-of-line-guards/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/sys/vm.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/sys/vm.py Fri Dec 3 13:33:30 2010 @@ -4,6 +4,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace from pypy.rlib.runicode import MAXUNICODE +from pypy.rlib import jit import sys # ____________________________________________________________ @@ -43,6 +44,7 @@ # directly from the C code in ceval.c, might be moved somewhere else. + at jit.dont_look_inside def setrecursionlimit(space, w_new_limit): """Set the maximum depth of the Python interpreter stack to n. This limit prevents infinite recursion from causing an overflow of the C From arigo at codespeak.net Fri Dec 3 13:41:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 3 Dec 2010 13:41:25 +0100 (CET) Subject: [pypy-svn] r79776 - pypy/trunk/pypy/jit/backend/x86/test Message-ID: <20101203124125.77559282BE9@codespeak.net> Author: arigo Date: Fri Dec 3 13:41:22 2010 New Revision: 79776 Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Log: Fix test. Modified: pypy/trunk/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_runner.py Fri Dec 3 13:41:22 2010 @@ -82,7 +82,7 @@ # relative addressing to work properly. addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.setup() + self.cpu.assembler.setup_once() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] From arigo at codespeak.net Fri Dec 3 13:46:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 3 Dec 2010 13:46:35 +0100 (CET) Subject: [pypy-svn] r79777 - pypy/trunk/lib_pypy/pypy_test Message-ID: <20101203124635.4176E282BEB@codespeak.net> Author: arigo Date: Fri Dec 3 13:46:31 2010 New Revision: 79777 Modified: pypy/trunk/lib_pypy/pypy_test/test_hashlib.py Log: Fix? I cannot reproduce the error locally Modified: pypy/trunk/lib_pypy/pypy_test/test_hashlib.py ============================================================================== --- pypy/trunk/lib_pypy/pypy_test/test_hashlib.py (original) +++ pypy/trunk/lib_pypy/pypy_test/test_hashlib.py Fri Dec 3 13:46:31 2010 @@ -41,7 +41,8 @@ # also test the pure Python implementation modname, constructor = pure_python_version[name].split('.') - builder = getattr(__import__(modname), constructor) + mod = __import__('lib_pypy.' + modname, None, None, ['__doc__']) + builder = getattr(mod, constructor) h = builder('') assert h.digest_size == expected_size assert h.digestsize == expected_size From arigo at codespeak.net Fri Dec 3 13:50:05 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 3 Dec 2010 13:50:05 +0100 (CET) Subject: [pypy-svn] r79778 - pypy/trunk/pypy/tool Message-ID: <20101203125005.8CEA2282BEB@codespeak.net> Author: arigo Date: Fri Dec 3 13:50:03 2010 New Revision: 79778 Modified: pypy/trunk/pypy/tool/error.py Log: Fix. Modified: pypy/trunk/pypy/tool/error.py ============================================================================== --- pypy/trunk/pypy/tool/error.py (original) +++ pypy/trunk/pypy/tool/error.py Fri Dec 3 13:50:03 2010 @@ -120,7 +120,7 @@ msg.append(" (%s getting at the binding!)" % ( e.__class__.__name__,)) return - for desc in descs.keys(): + for desc in list(descs): func = desc.pyobj if func is None: r = repr(desc) From afa at codespeak.net Fri Dec 3 13:54:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 13:54:51 +0100 (CET) Subject: [pypy-svn] r79779 - pypy/trunk/pypy/rpython/tool Message-ID: <20101203125451.0B72F282BEC@codespeak.net> Author: afa Date: Fri Dec 3 13:54:49 2010 New Revision: 79779 Modified: pypy/trunk/pypy/rpython/tool/rffi_platform.py Log: There is no uint64_t anywhere in PyPy; remove which is not supported on all platforms. Modified: pypy/trunk/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/trunk/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/trunk/pypy/rpython/tool/rffi_platform.py Fri Dec 3 13:54:49 2010 @@ -639,7 +639,6 @@ C_HEADER = """ #include #include /* for offsetof() */ -#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); From david at codespeak.net Fri Dec 3 16:31:31 2010 From: david at codespeak.net (david at codespeak.net) Date: Fri, 3 Dec 2010 16:31:31 +0100 (CET) Subject: [pypy-svn] r79780 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . helper test Message-ID: <20101203153131.EFAD4282B9D@codespeak.net> Author: david Date: Fri Dec 3 16:31:28 2010 New Revision: 79780 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_assembler.py Log: Refactoring and fixes for arithmetic operations Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Fri Dec 3 16:31:28 2010 @@ -35,13 +35,16 @@ self.malloc_unicode_func_addr = 0 self.memcpy_addr = 0 + def setup_mc(self): + self.mc = ARMv7Builder() + self._exit_code_addr = self.mc.curraddr() + self._gen_exit_path() + self.align() + self.mc._start_addr = self.mc.curraddr() + def setup(self): if self.mc is None: - self.mc = ARMv7Builder() - self._exit_code_addr = self.mc.curraddr() - self._gen_exit_path() - self.align() - self.mc._start_addr = self.mc.curraddr() + self.setup_mc() # Addresses of functions called by new_xxx operations gc_ll_descr = self.cpu.gc_ll_descr @@ -445,16 +448,21 @@ return arg.getint() <= size and lower_bound return False - def _ensure_value_is_boxed(self, thing, regalloc): + def _ensure_value_is_boxed(self, thing, regalloc, forbidden_vars=[]): box = None loc = None if isinstance(thing, Const): - box = TempBox() - loc = regalloc.force_allocate_reg(box) + if isinstance(thing, ConstInt): + box = BoxInt() + else: + box = TempBox() + loc = regalloc.force_allocate_reg(box, + forbidden_vars=forbidden_vars) imm = regalloc.convert_to_imm(thing) self.mc.gen_load_int(loc.value, imm.getint()) else: - loc = regalloc.make_sure_var_in_reg(thing, imm_fine=False) + loc = regalloc.make_sure_var_in_reg(thing, + forbidden_vars=forbidden_vars, imm_fine=False) box = thing return loc, box Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py Fri Dec 3 16:31:28 2010 @@ -1,47 +1,53 @@ from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r +from pypy.jit.backend.arm.codebuilder import AbstractARMv7Builder from pypy.jit.metainterp.history import ConstInt, BoxInt def gen_emit_op_unary_cmp(true_cond, false_cond): def f(self, op, regalloc, fcond): assert fcond is not None a0 = op.getarg(0) - reg = regalloc.make_sure_var_in_reg(a0, imm_fine=False) - res = regalloc.force_allocate_reg(op.result, [a0]) + reg, box = self._ensure_value_is_boxed(a0, regalloc) + res = regalloc.force_allocate_reg(op.result, [box]) + regalloc.possibly_free_vars([a0, box, op.result]) + self.mc.CMP_ri(reg.value, 0) self.mc.MOV_ri(res.value, 1, true_cond) self.mc.MOV_ri(res.value, 0, false_cond) - regalloc.possibly_free_var(a0) - regalloc.possibly_free_var(op.result) return fcond return f def gen_emit_op_ri(opname, imm_size=0xFF, commutative=True, allow_zero=True): + ri_op = getattr(AbstractARMv7Builder, '%s_ri' % opname) + rr_op = getattr(AbstractARMv7Builder, '%s_rr' % opname) def f(self, op, regalloc, fcond): assert fcond is not None - ri_op = getattr(self.mc, '%s_ri' % opname) - rr_op = getattr(self.mc, '%s_rr' % opname) + a0 = op.getarg(0) + a1 = op.getarg(1) + boxes = list(op.getarglist()) + imm_a0 = self._check_imm_arg(a0, imm_size, allow_zero=allow_zero) + imm_a1 = self._check_imm_arg(a1, imm_size, allow_zero=allow_zero) + if not imm_a0 and imm_a1: + l0, box = self._ensure_value_is_boxed(a0, regalloc) + boxes.append(box) + l1 = regalloc.make_sure_var_in_reg(a1, boxes) + elif commutative and imm_a0 and not imm_a1: + l1 = regalloc.make_sure_var_in_reg(a0, boxes) + l0, box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(box) + else: + l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) + boxes.append(box) + l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(box) + res = regalloc.force_allocate_reg(op.result, boxes) + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) - arg0 = op.getarg(0) - arg1 = op.getarg(1) - imm_a0 = self._check_imm_arg(arg0, imm_size, allow_zero=allow_zero) - imm_a1 = self._check_imm_arg(arg1, imm_size, allow_zero=allow_zero) - if commutative and imm_a0: - l0 = regalloc.make_sure_var_in_reg(arg0, [arg1], imm_fine=imm_a0) - l1 = regalloc.make_sure_var_in_reg(arg1, [arg0]) - res = regalloc.force_allocate_reg(op.result, [arg0, arg1]) - ri_op(res.value, l1.value, imm=l0.getint(), cond=fcond) - elif imm_a1: - l0 = regalloc.make_sure_var_in_reg(arg0, [arg1], imm_fine=False) - l1 = regalloc.make_sure_var_in_reg(arg1, [arg0], imm_fine=True) - res = regalloc.force_allocate_reg(op.result, [arg0, arg1]) - ri_op(res.value, l0.value, imm=l1.getint(), cond=fcond) + if l1.is_imm(): + ri_op(self.mc, res.value, l0.value, imm=l1.getint(), cond=fcond) else: - l0 = regalloc.make_sure_var_in_reg(arg0, [arg1], imm_fine=False) - l1 = regalloc.make_sure_var_in_reg(arg1, [arg0], imm_fine=False) - res = regalloc.force_allocate_reg(op.result, [arg0, arg1]) - rr_op(res.value, l0.value, l1.value) - regalloc.possibly_free_vars([arg0, arg1, op.result]) + rr_op(self.mc, res.value, l0.value, l1.value) return fcond return f Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Fri Dec 3 16:31:28 2010 @@ -29,30 +29,35 @@ _mixin_ = True def emit_op_int_add(self, op, regalloc, fcond): - # assuming only one argument is constant + #XXX check if neg values are supported for imm values a0 = op.getarg(0) a1 = op.getarg(1) - imm_a0 = isinstance(a0, ConstInt) and (a0.getint() <= 0xFF or -1 * a0.getint() <= 0xFF) - imm_a1 = isinstance(a1, ConstInt) and (a1.getint() <= 0xFF or -1 * a1.getint() <= 0xFF) - if imm_a0: - imm_a0, imm_a1 = imm_a1, imm_a0 - a0, a1 = a1, a0 - if imm_a1: - l0 = regalloc.make_sure_var_in_reg(a0, imm_fine=False) - l1 = regalloc.make_sure_var_in_reg(a1, [a0], imm_fine=True) - res = regalloc.force_allocate_reg(op.result, [a0, a1]) - if l1.getint() < 0: - self.mc.SUB_ri(res.value, l0.value, -1 * l1.getint(), s=1) - else: - self.mc.ADD_ri(res.value, l0.value, l1.getint(), s=1) + boxes = list(op.getarglist()) + imm_a0 = self._check_imm_arg(a0) + imm_a1 = self._check_imm_arg(a1) + if not imm_a0 and imm_a1: + l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) + l1 = regalloc.make_sure_var_in_reg(a1, [a0]) + boxes.append(box) + elif imm_a0 and not imm_a1: + l0 = regalloc.make_sure_var_in_reg(a0) + l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(box) + else: + l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) + boxes.append(box) + l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(box) + res = regalloc.force_allocate_reg(op.result, boxes) + + if l0.is_imm(): + self.mc.ADD_ri(res.value, l1.value, imm=l0.value, s=1) + elif l1.is_imm(): + self.mc.ADD_ri(res.value, l0.value, imm=l1.value, s=1) else: - l0 = regalloc.make_sure_var_in_reg(a0, imm_fine=False) - l1 = regalloc.make_sure_var_in_reg(a1, forbidden_vars=[a0], imm_fine=False) - res = regalloc.force_allocate_reg(op.result, forbidden_vars=[a0, a1]) self.mc.ADD_rr(res.value, l0.value, l1.value, s=1) - regalloc.possibly_free_var(a0) - regalloc.possibly_free_var(a1) + regalloc.possibly_free_vars(boxes) regalloc.possibly_free_var(op.result) return fcond @@ -96,11 +101,16 @@ def emit_op_int_mul(self, op, regalloc, fcond): a0 = op.getarg(0) a1 = op.getarg(1) - reg1 = regalloc.make_sure_var_in_reg(a0, [a1], imm_fine=False) - reg2 = regalloc.make_sure_var_in_reg(a1, [a0], imm_fine=False) - res = regalloc.force_allocate_reg(op.result, [a0, a1]) + boxes = list(op.getarglist()) + + reg1, box = self._ensure_value_is_boxed(a0, regalloc, forbidden_vars=boxes) + boxes.append(box) + reg2, box = self._ensure_value_is_boxed(a1, regalloc, forbidden_vars=boxes) + boxes.append(box) + + res = regalloc.force_allocate_reg(op.result, boxes) self.mc.MUL(res.value, reg1.value, reg2.value) - regalloc.possibly_free_vars_for_op(op) + regalloc.possibly_free_vars(boxes) regalloc.possibly_free_var(op.result) return fcond @@ -108,18 +118,25 @@ def emit_guard_int_mul_ovf(self, op, guard, regalloc, fcond): a0 = op.getarg(0) a1 = op.getarg(1) - reg1 = regalloc.make_sure_var_in_reg(a0, [a1], imm_fine=False) - reg2 = regalloc.make_sure_var_in_reg(a1, [a0], imm_fine=False) - res = regalloc.force_allocate_reg(op.result, [a0, a1]) + boxes = list(op.getarglist()) + + reg1, box = self._ensure_value_is_boxed(a0, regalloc, forbidden_vars=boxes) + boxes.append(box) + reg2, box = self._ensure_value_is_boxed(a1, regalloc, forbidden_vars=boxes) + boxes.append(box) + res = regalloc.force_allocate_reg(op.result, boxes) + self.mc.SMULL(res.value, r.ip.value, reg1.value, reg2.value, cond=fcond) self.mc.CMP_rr(r.ip.value, res.value, shifttype=shift.ASR, imm=31, cond=fcond) - regalloc.possibly_free_vars_for_op(op) - if op.result: - regalloc.possibly_free_var(op.result) + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) + if guard.getopnum() == rop.GUARD_OVERFLOW: return self._emit_guard(guard, regalloc, c.NE) - else: + if guard.getopnum() == rop.GUARD_NO_OVERFLOW: return self._emit_guard(guard, regalloc, c.EQ) + else: + assert 0 emit_op_int_floordiv = gen_emit_op_by_helper_call('DIV') emit_op_int_mod = gen_emit_op_by_helper_call('MOD') Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_assembler.py Fri Dec 3 16:31:28 2010 @@ -15,6 +15,7 @@ class TestRunningAssembler(): def setup_method(self, method): self.a = AssemblerARM(None) + self.a.setup_mc() def test_make_operation_list(self): i = rop.INT_ADD From david at codespeak.net Fri Dec 3 16:35:18 2010 From: david at codespeak.net (david at codespeak.net) Date: Fri, 3 Dec 2010 16:35:18 +0100 (CET) Subject: [pypy-svn] r79781 - pypy/branch/arm-backend/pypy/jit/backend/arm/test Message-ID: <20101203153518.23225282BE7@codespeak.net> Author: david Date: Fri Dec 3 16:35:16 2010 New Revision: 79781 Added: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py Log: Import tests based on failing random tests Added: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py ============================================================================== --- (empty file) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py Fri Dec 3 16:35:16 2010 @@ -0,0 +1,496 @@ +import py +from pypy.jit.metainterp.history import (AbstractFailDescr, + AbstractDescr, + BasicFailDescr, + BoxInt, Box, BoxPtr, + LoopToken, + ConstInt, ConstPtr, + BoxObj, Const, + ConstObj, BoxFloat, ConstFloat) +from pypy.jit.metainterp.resoperation import ResOperation, rop +from pypy.rpython.test.test_llinterp import interpret +from pypy.jit.backend.detect_cpu import getcpuclass + +CPU = getcpuclass() +class TestStuff(object): + + def test0(self): + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_SUB, [ConstInt(-1073741824), v7], v11), + ResOperation(rop.INT_GE, [v3, ConstInt(23)], v12), + ResOperation(rop.GUARD_TRUE, [v12], None, descr=faildescr1), + ResOperation(rop.FINISH, [v9, v6, v10, v2, v8, v5, v1, v4], None, descr=faildescr2), + ] + looptoken = LoopToken() + operations[2].setfailargs([v12, v8, v3, v2, v1, v11]) + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, -12) + cpu.set_future_value_int(1, -26) + cpu.set_future_value_int(2, -19) + cpu.set_future_value_int(3, 7) + cpu.set_future_value_int(4, -5) + cpu.set_future_value_int(5, -24) + cpu.set_future_value_int(6, -37) + cpu.set_future_value_int(7, 62) + cpu.set_future_value_int(8, 9) + cpu.set_future_value_int(9, 12) + op = cpu.execute_token(looptoken) + assert cpu.get_latest_value_int(0) == 0 + assert cpu.get_latest_value_int(1) == 62 + assert cpu.get_latest_value_int(2) == -19 + assert cpu.get_latest_value_int(3) == -26 + assert cpu.get_latest_value_int(4) == -12 + assert cpu.get_latest_value_int(5) == -1073741787 + + def test_overflow(self): + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + v16 = BoxInt() + v17 = BoxInt() + v18 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_SUB, [ConstInt(21), v5], v11), + ResOperation(rop.INT_MUL_OVF, [v8, v4], v12), + ResOperation(rop.GUARD_NO_OVERFLOW, [], None, descr=faildescr1), + ResOperation(rop.UINT_LT, [v10, v3], v13), + ResOperation(rop.INT_IS_TRUE, [v3], v14), + ResOperation(rop.INT_XOR, [v9, v8], v15), + ResOperation(rop.INT_LE, [v12, v6], v16), + ResOperation(rop.UINT_GT, [v15, v5], v17), + ResOperation(rop.UINT_LE, [ConstInt(-9), v13], v18), + ResOperation(rop.GUARD_FALSE, [v13], None, descr=faildescr2), + ResOperation(rop.FINISH, [v7, v1, v2], None, descr=faildescr3), + ] + operations[2].setfailargs([v10, v6]) + operations[9].setfailargs([v15, v7, v10, v18, v4, v17, v1]) + looptoken = LoopToken() + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, 16) + cpu.set_future_value_int(1, 5) + cpu.set_future_value_int(2, 5) + cpu.set_future_value_int(3, 16) + cpu.set_future_value_int(4, 46) + cpu.set_future_value_int(5, 6) + cpu.set_future_value_int(6, 63) + cpu.set_future_value_int(7, 39) + cpu.set_future_value_int(8, 78) + cpu.set_future_value_int(9, 0) + op = cpu.execute_token(looptoken) + assert cpu.get_latest_value_int(0) == 105 + assert cpu.get_latest_value_int(1) == 63 + assert cpu.get_latest_value_int(2) == 0 + assert cpu.get_latest_value_int(3) == 0 + assert cpu.get_latest_value_int(4) == 16 + assert cpu.get_latest_value_int(5) == 1 + assert cpu.get_latest_value_int(6) == 16 + + def test_sub_with_neg_const_first_arg(self): + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + tmp13 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_EQ, [ConstInt(17), v9], v11), + ResOperation(rop.INT_SUB_OVF, [ConstInt(-32), v7], v12), + ResOperation(rop.GUARD_NO_OVERFLOW, [], None, descr=faildescr1), + ResOperation(rop.INT_IS_ZERO, [v12], tmp13), + ResOperation(rop.GUARD_TRUE, [tmp13], None, descr=faildescr2), + ResOperation(rop.FINISH, [v5, v2, v1, v10, v3, v8, v4, v6], None, descr=faildescr3) + ] + operations[2].setfailargs([v8, v3]) + operations[4].setfailargs([v2, v12, v1, v3, v4]) + looptoken = LoopToken() + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, -5) + cpu.set_future_value_int(1, 24) + cpu.set_future_value_int(2, 46) + cpu.set_future_value_int(3, -15) + cpu.set_future_value_int(4, 13) + cpu.set_future_value_int(5, -8) + cpu.set_future_value_int(6, 0) + cpu.set_future_value_int(7, -6) + cpu.set_future_value_int(8, 6) + cpu.set_future_value_int(9, 6) + op = cpu.execute_token(looptoken) + assert op.identifier == 2 + assert cpu.get_latest_value_int(0) == 24 + assert cpu.get_latest_value_int(1) == -32 + assert cpu.get_latest_value_int(2) == -5 + assert cpu.get_latest_value_int(3) == 46 + assert cpu.get_latest_value_int(4) == -15 + + def test_tempbox_spilling_in_sub(self): + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_LT, [v9, v9], v11), + ResOperation(rop.INT_ADD, [ConstInt(715827882), v4], v12), + ResOperation(rop.INT_NEG, [v11], v13), + ResOperation(rop.INT_IS_TRUE, [v3], v14), + ResOperation(rop.INT_SUB_OVF, [v3, ConstInt(-95)], v15), + ResOperation(rop.GUARD_NO_OVERFLOW, [], None, descr=faildescr1), + ResOperation(rop.FINISH, [v8, v2, v6, v5, v7, v1, v10], None, descr=faildescr2), + ] + operations[5].setfailargs([]) + looptoken = LoopToken() + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, 19) + cpu.set_future_value_int(1, -3) + cpu.set_future_value_int(2, -58) + cpu.set_future_value_int(3, -7) + cpu.set_future_value_int(4, 12) + cpu.set_future_value_int(5, 22) + cpu.set_future_value_int(6, -54) + cpu.set_future_value_int(7, -29) + cpu.set_future_value_int(8, -19) + cpu.set_future_value_int(9, -64) + op = cpu.execute_token(looptoken) + assert cpu.get_latest_value_int(0) == -29 + assert cpu.get_latest_value_int(1) == -3 + assert cpu.get_latest_value_int(2) == 22 + assert cpu.get_latest_value_int(3) == 12 + assert cpu.get_latest_value_int(4) == -54 + assert cpu.get_latest_value_int(5) == 19 + assert cpu.get_latest_value_int(6) == -64 + + def test_tempbox2(self): + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_LT, [v5, ConstInt(-67)], v11), + ResOperation(rop.INT_INVERT, [v2], v12), + ResOperation(rop.INT_SUB, [ConstInt(-45), v2], v13), + ResOperation(rop.INT_SUB, [ConstInt(99), v6], v14), + ResOperation(rop.INT_MUL_OVF, [v6, v9], v15), + ResOperation(rop.GUARD_NO_OVERFLOW, [], None, descr=faildescr1), + ResOperation(rop.FINISH, [v1, v4, v10, v8, v7, v3], None, descr=faildescr2), + ] + looptoken = LoopToken() + operations[5].setfailargs([]) + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, 1073741824) + cpu.set_future_value_int(1, 95) + cpu.set_future_value_int(2, -16) + cpu.set_future_value_int(3, 5) + cpu.set_future_value_int(4, 92) + cpu.set_future_value_int(5, 12) + cpu.set_future_value_int(6, 32) + cpu.set_future_value_int(7, 17) + cpu.set_future_value_int(8, 37) + cpu.set_future_value_int(9, -63) + op = cpu.execute_token(looptoken) + assert cpu.get_latest_value_int(0) == 1073741824 + assert cpu.get_latest_value_int(1) == 5 + assert cpu.get_latest_value_int(2) == -63 + assert cpu.get_latest_value_int(3) == 17 + assert cpu.get_latest_value_int(4) == 32 + assert cpu.get_latest_value_int(5) == -16 + + def test_wrong_guard(self): + # generated by: + # ../test/ test/test_zll_random.py -l -k arm -s --block-length=10 --random-seed=4338 + + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) + faildescr4 = BasicFailDescr(4) + v1 = BoxInt(32) + v2 = BoxInt(41) + v3 = BoxInt(-9) + v4 = BoxInt(12) + v5 = BoxInt(-18) + v6 = BoxInt(46) + v7 = BoxInt(15) + v8 = BoxInt(17) + v9 = BoxInt(10) + v10 = BoxInt(12) + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + tmp15 = BoxInt() + tmp16 = BoxInt() + tmp17 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_IS_TRUE, [v1], tmp15), + ResOperation(rop.GUARD_TRUE, [tmp15], None, descr=faildescr1), + ResOperation(rop.INT_GT, [v4, v5], v11), + ResOperation(rop.INT_XOR, [ConstInt(-4), v7], v12), + ResOperation(rop.INT_MUL, [ConstInt(23), v11], v13), + ResOperation(rop.UINT_GE, [ConstInt(1), v13], v14), + ResOperation(rop.INT_IS_ZERO, [v14], tmp16), + ResOperation(rop.GUARD_TRUE, [tmp16], None, descr=faildescr2), + ResOperation(rop.INT_IS_TRUE, [v12], tmp17), + ResOperation(rop.GUARD_FALSE, [tmp17], None, descr=faildescr3), + ResOperation(rop.FINISH, [v8, v10, v6, v3, v2, v9], None, descr=faildescr4), + ] + looptoken = LoopToken() + operations[1].setfailargs([v8, v6, v1]) + operations[7].setfailargs([v4]) + operations[9].setfailargs([v10, v13]) + cpu.set_future_value_int(0, 32) + cpu.set_future_value_int(1, 41) + cpu.set_future_value_int(2, -9) + cpu.set_future_value_int(3, 12) + cpu.set_future_value_int(4, -18) + cpu.set_future_value_int(5, 46) + cpu.set_future_value_int(6, 15) + cpu.set_future_value_int(7, 17) + cpu.set_future_value_int(8, 10) + cpu.set_future_value_int(9, 12) + cpu.compile_loop(inputargs, operations, looptoken) + op = cpu.execute_token(looptoken) + assert op.identifier == 3 + assert cpu.get_latest_value_int(0) == 12 + assert cpu.get_latest_value_int(1) == 23 + + def test_wrong_guard2(self): + # random seed: 8029 + # block length: 10 + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + v16 = BoxInt() + tmp17 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_ADD_OVF, [v8, ConstInt(-30)], v11), + ResOperation(rop.GUARD_NO_OVERFLOW, [], None, descr=faildescr1), + ResOperation(rop.UINT_LE, [v11, v1], v12), + ResOperation(rop.INT_AND, [v11, ConstInt(31)], tmp17), + ResOperation(rop.UINT_RSHIFT, [v12, tmp17], v13), + ResOperation(rop.INT_NE, [v3, v2], v14), + ResOperation(rop.INT_NE, [ConstInt(1), v11], v15), + ResOperation(rop.INT_NE, [ConstInt(23), v15], v16), + ResOperation(rop.GUARD_FALSE, [v15], None, descr=faildescr2), + ResOperation(rop.FINISH, [v4, v10, v6, v5, v9, v7], None, descr=faildescr3), + ] + operations[1].setfailargs([v6, v8, v1, v4]) + operations[8].setfailargs([v5, v9]) + looptoken = LoopToken() + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, -8) + cpu.set_future_value_int(1, 0) + cpu.set_future_value_int(2, 62) + cpu.set_future_value_int(3, 35) + cpu.set_future_value_int(4, 16) + cpu.set_future_value_int(5, 9) + cpu.set_future_value_int(6, 30) + cpu.set_future_value_int(7, 581610154) + cpu.set_future_value_int(8, -1) + cpu.set_future_value_int(9, 738197503) + op = cpu.execute_token(looptoken) + assert op.identifier == 2 + assert cpu.get_latest_value_int(0) == 16 + assert cpu.get_latest_value_int(1) == -1 + def test_wrong_result(self): + # generated by: + # ../test/ test/test_zll_random.py -l -k arm -s --block-length=10 --random-seed=7389 + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) + faildescr4 = BasicFailDescr(4) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + tmp16 = BoxInt() + tmp17 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_IS_TRUE, [v3], tmp16), + ResOperation(rop.GUARD_TRUE, [tmp16], None, descr=faildescr1), + ResOperation(rop.INT_AND, [v7, ConstInt(31)], tmp17), + ResOperation(rop.INT_RSHIFT, [v5, tmp17], v11), + ResOperation(rop.INT_OR, [v6, v8], v12), + ResOperation(rop.GUARD_VALUE, [v11, ConstInt(-2)], None, descr=faildescr2), + ResOperation(rop.INT_LE, [ConstInt(1789569706), v10], v13), + ResOperation(rop.INT_IS_TRUE, [v4], v14), + ResOperation(rop.INT_XOR, [v14, v3], v15), + ResOperation(rop.GUARD_VALUE, [v8, ConstInt(-8)], None, descr=faildescr3), + ResOperation(rop.FINISH, [v1, v2, v9], None, descr=faildescr4), + ] + operations[1].setfailargs([v9, v1]) + operations[5].setfailargs([v10, v2, v11, v3]) + operations[9].setfailargs([v5, v7, v12, v14, v2, v13, v8]) + looptoken = LoopToken() + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, 0) + cpu.set_future_value_int(1, -2) + cpu.set_future_value_int(2, 24) + cpu.set_future_value_int(3, 1) + cpu.set_future_value_int(4, -4) + cpu.set_future_value_int(5, 13) + cpu.set_future_value_int(6, -95) + cpu.set_future_value_int(7, 33) + cpu.set_future_value_int(8, 2) + cpu.set_future_value_int(9, -44) + op = cpu.execute_token(looptoken) + assert op.identifier == 3 + assert cpu.get_latest_value_int(0) == -4 + assert cpu.get_latest_value_int(1) == -95 + assert cpu.get_latest_value_int(2) == 45 + assert cpu.get_latest_value_int(3) == 1 + assert cpu.get_latest_value_int(4) == -2 + assert cpu.get_latest_value_int(5) == 0 + assert cpu.get_latest_value_int(6) == 33 + + def test_int_add(self): + # random seed: 1202 + # block length: 4 + # AssertionError: Got 1431655764, expected 357913940 for value #3 + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + tmp12 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_ADD, [ConstInt(-1073741825), v3], v11), + ResOperation(rop.INT_IS_TRUE, [v1], tmp12), + ResOperation(rop.GUARD_FALSE, [tmp12], None, descr=faildescr1), + ResOperation(rop.FINISH, [v8, v2, v10, v6, v7, v9, v5, v4], None, descr=faildescr2), + ] + operations[2].setfailargs([v10, v3, v6, v11, v9, v2]) + looptoken = LoopToken() + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, 3) + cpu.set_future_value_int(1, -5) + cpu.set_future_value_int(2, 1431655765) + cpu.set_future_value_int(3, 47) + cpu.set_future_value_int(4, 12) + cpu.set_future_value_int(5, 1789569706) + cpu.set_future_value_int(6, 15) + cpu.set_future_value_int(7, 939524096) + cpu.set_future_value_int(8, 16) + cpu.set_future_value_int(9, -43) + op = cpu.execute_token(looptoken) + assert op.identifier == 1 + assert cpu.get_latest_value_int(0) == -43 + assert cpu.get_latest_value_int(1) == 1431655765 + assert cpu.get_latest_value_int(2) == 1789569706 + assert cpu.get_latest_value_int(3) == 357913940 + assert cpu.get_latest_value_int(4) == 16 + assert cpu.get_latest_value_int(5) == -5 From antocuni at codespeak.net Fri Dec 3 17:00:13 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Fri, 3 Dec 2010 17:00:13 +0100 (CET) Subject: [pypy-svn] r79782 - in pypy/branch/jitypes2: lib_pypy/_ctypes pypy/module/_ffi pypy/module/_ffi/test pypy/rlib Message-ID: <20101203160013.04AD8282B9D@codespeak.net> Author: antocuni Date: Fri Dec 3 17:00:12 2010 New Revision: 79782 Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py pypy/branch/jitypes2/pypy/rlib/libffi.py Log: cast pointers to unsigned long, not signed. This make it possible to use app-level longs to represent them in _ffi Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Fri Dec 3 17:00:12 2010 @@ -199,7 +199,7 @@ restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype, thisarg) result = funcptr(*newargs) - if restype._ffishape == 'u': + if restype and restype._ffishape == 'u': # XXX: maybe it's a job of _ffi? result = unichr(result) ## resbuffer = funcptr(*[arg._get_buffer_for_param()._buffer @@ -384,14 +384,25 @@ return wrapped_args def _unwrap_args(self, argtypes, args): + """ + Convert from ctypes high-level values to low-level values suitables to + be passed to _ffi + """ assert len(argtypes) == len(args) newargs = [] for argtype, arg in zip(argtypes, args): - value = arg.value if argtype._ffishape == 'u': # XXX: who should do this conversion? Maybe _ffi? + value = arg.value assert isinstance(value, basestring) and len(value) == 1 value = ord(value) + elif argtype._ffishape == 'P': + value = arg._buffer.buffer + if value > sys.maxint: + # XXX: simulate overflow so that _ffi receive and int, not a long + value = (-sys.maxint-1)*2 + value + else: + value = arg.value newargs.append(value) return newargs Modified: pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py (original) +++ pypy/branch/jitypes2/pypy/module/_ffi/interp_ffi.py Fri Dec 3 17:00:12 2010 @@ -124,8 +124,6 @@ intres = rffi.cast(rffi.LONG, call(argchain, rffi.SHORT)) elif restype is libffi.types.schar: intres = rffi.cast(rffi.LONG, call(argchain, rffi.SIGNEDCHAR)) - elif restype is libffi.types.pointer: - intres = rffi.cast(rffi.LONG, call(argchain, rffi.VOIDP)) else: raise OperationError(space.w_ValueError, space.wrap('Unsupported restype')) @@ -149,6 +147,9 @@ # special case uintres = call(argchain, rffi.ULONG) return space.wrap(uintres) + elif restype is libffi.types.pointer: + uintres = rffi.cast(rffi.ULONG, call(argchain, rffi.VOIDP)) + return space.wrap(uintres) elif restype is libffi.types.uint: intres = rffi.cast(rffi.LONG, call(argchain, rffi.UINT)) elif restype is libffi.types.ushort: Modified: pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py (original) +++ pypy/branch/jitypes2/pypy/module/_ffi/test/test__ffi.py Fri Dec 3 17:00:12 2010 @@ -124,6 +124,17 @@ assert get_dummy() == 123 set_val_to_ptr(ptr, 0) + def test_huge_pointer_args(self): + """ + #include + long is_null_ptr(void* ptr) { return ptr == NULL; } + """ + import sys + from _ffi import CDLL, types + libfoo = CDLL(self.libfoo_name) + is_null_ptr = libfoo.getfunc('is_null_ptr', [types.pointer], types.ulong) + assert not is_null_ptr(sys.maxint+1) + def test_unsigned_long_args(self): """ unsigned long sum_xy_ul(unsigned long x, unsigned long y) Modified: pypy/branch/jitypes2/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/jitypes2/pypy/rlib/libffi.py (original) +++ pypy/branch/jitypes2/pypy/rlib/libffi.py Fri Dec 3 17:00:12 2010 @@ -41,7 +41,7 @@ if ffi_type is types.void: return 'v' elif ffi_type is types.double: return 'f' elif ffi_type is types.float: return 's' - elif ffi_type is types.pointer: return 'i' + elif ffi_type is types.pointer: return 'u' # elif ffi_type is types.schar: return 'i' elif ffi_type is types.uchar: return 'u' From afa at codespeak.net Fri Dec 3 17:00:36 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 17:00:36 +0100 (CET) Subject: [pypy-svn] r79783 - in pypy/branch/fast-forward/pypy/module/cpyext: . test Message-ID: <20101203160036.D90DE282BDD@codespeak.net> Author: afa Date: Fri Dec 3 17:00:35 2010 New Revision: 79783 Modified: pypy/branch/fast-forward/pypy/module/cpyext/dictobject.py pypy/branch/fast-forward/pypy/module/cpyext/stubs.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_dictobject.py Log: Add PyDict_Contains Modified: pypy/branch/fast-forward/pypy/module/cpyext/dictobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/dictobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/dictobject.py Fri Dec 3 17:00:35 2010 @@ -67,6 +67,15 @@ len(p) on a dictionary.""" return space.int_w(space.len(w_obj)) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PyDict_Contains(space, w_obj, w_value): + """Determine if dictionary p contains key. If an item in p is matches + key, return 1, otherwise return 0. On error, return -1. + This is equivalent to the Python expression key in p. + """ + w_res = space.contains(w_obj, w_value) + return space.int_w(w_res) + @cpython_api([PyObject], lltype.Void) def PyDict_Clear(space, w_obj): """Empty an existing dictionary of all key-value pairs.""" Modified: pypy/branch/fast-forward/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/stubs.py Fri Dec 3 17:00:35 2010 @@ -615,14 +615,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PyDict_Contains(space, p, key): - """Determine if dictionary p contains key. If an item in p is matches - key, return 1, otherwise return 0. On error, return -1. - This is equivalent to the Python expression key in p. - """ - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyDict_DelItemString(space, p, key): """Remove the entry in dictionary p which has a key specified by the string Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_dictobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_dictobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_dictobject.py Fri Dec 3 17:00:35 2010 @@ -29,6 +29,9 @@ rffi.free_charp(buf) assert not api.PyErr_Occurred() + assert api.PyDict_Contains(d, space.wrap("c")) + assert not api.PyDict_Contains(d, space.wrap("z")) + assert api.PyDict_DelItem(d, space.wrap("c")) == 0 assert api.PyDict_DelItem(d, space.wrap("name")) < 0 assert api.PyErr_Occurred() is space.w_KeyError From david at codespeak.net Fri Dec 3 17:29:20 2010 From: david at codespeak.net (david at codespeak.net) Date: Fri, 3 Dec 2010 17:29:20 +0100 (CET) Subject: [pypy-svn] r79784 - pypy/branch/arm-backend/pypy/jit/backend/arm/helper Message-ID: <20101203162920.99E83282B9E@codespeak.net> Author: david Date: Fri Dec 3 17:29:17 2010 New Revision: 79784 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py Log: Fix comparison ops based on random test results Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py Fri Dec 3 17:29:17 2010 @@ -45,7 +45,7 @@ regalloc.possibly_free_var(op.result) if l1.is_imm(): - ri_op(self.mc, res.value, l0.value, imm=l1.getint(), cond=fcond) + ri_op(self.mc, res.value, l0.value, imm=l1.value, cond=fcond) else: rr_op(self.mc, res.value, l0.value, l1.value) return fcond @@ -74,30 +74,32 @@ def gen_emit_cmp_op(condition, inverse=False): def f(self, op, regalloc, fcond): assert fcond is not None - args = op.getarglist() + boxes = list(op.getarglist()) if not inverse: - arg0 = op.getarg(0) - arg1 = op.getarg(1) + arg0, arg1 = boxes else: - arg0 = op.getarg(1) - arg1 = op.getarg(0) + arg1, arg0 = boxes # XXX consider swapping argumentes if arg0 is const imm_a0 = self._check_imm_arg(arg0) imm_a1 = self._check_imm_arg(arg1) + + l0, box = self._ensure_value_is_boxed(arg0, regalloc, forbidden_vars=boxes) + boxes.append(box) if imm_a1 and not imm_a0: - l0 = regalloc.make_sure_var_in_reg(arg0, args, imm_fine=False) - l1 = regalloc.make_sure_var_in_reg(arg1, args) - res = regalloc.force_allocate_reg(op.result) - self.mc.CMP_ri(l0.value, imm=l1.getint(), cond=fcond) + l1 = regalloc.make_sure_var_in_reg(arg1, boxes) else: - l0 = regalloc.make_sure_var_in_reg(arg0, args, imm_fine=False) - l1 = regalloc.make_sure_var_in_reg(arg1, args, imm_fine=False) - res = regalloc.force_allocate_reg(op.result) - self.mc.CMP_rr(l0.value, l1.value, cond=fcond) + l1, box = self._ensure_value_is_boxed(arg1, regalloc, forbidden_vars=boxes) + boxes.append(box) + res = regalloc.force_allocate_reg(op.result) + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) inv = c.get_opposite_of(condition) + if l1.is_imm(): + self.mc.CMP_ri(l0.value, imm=l1.getint(), cond=fcond) + else: + self.mc.CMP_rr(l0.value, l1.value, cond=fcond) self.mc.MOV_ri(res.value, 1, cond=condition) self.mc.MOV_ri(res.value, 0, cond=inv) - regalloc.possibly_free_vars([arg0, arg1, op.result]) return fcond return f From afa at codespeak.net Fri Dec 3 17:56:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 17:56:22 +0100 (CET) Subject: [pypy-svn] r79785 - in pypy/branch/fast-forward/pypy/module/cpyext: . test Message-ID: <20101203165622.89EE4282BE0@codespeak.net> Author: afa Date: Fri Dec 3 17:56:20 2010 New Revision: 79785 Modified: pypy/branch/fast-forward/pypy/module/cpyext/api.py pypy/branch/fast-forward/pypy/module/cpyext/complexobject.py pypy/branch/fast-forward/pypy/module/cpyext/methodobject.py pypy/branch/fast-forward/pypy/module/cpyext/modsupport.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_methodobject.py pypy/branch/fast-forward/pypy/module/cpyext/typeobject.py pypy/branch/fast-forward/pypy/module/cpyext/typeobjectdefs.py Log: Implement PyCFunction_GetFunction(). Works only with functions defined in extension modules, not with pypy built-in functions! Modified: pypy/branch/fast-forward/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/api.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/api.py Fri Dec 3 17:56:20 2010 @@ -193,7 +193,7 @@ the API headers. """ if error is _NOT_SPECIFIED: - if restype is PyObject: + if isinstance(restype, lltype.Ptr): error = lltype.nullptr(restype.TO) elif restype is lltype.Void: error = CANNOT_FAIL Modified: pypy/branch/fast-forward/pypy/module/cpyext/complexobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/complexobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/complexobject.py Fri Dec 3 17:56:20 2010 @@ -36,7 +36,7 @@ # lltype does not handle functions returning a structure. This implements a # helper function, which takes as argument a reference to the return value. - at cpython_api([PyObject, Py_complex_ptr], lltype.Void, error=None) + at cpython_api([PyObject, Py_complex_ptr], lltype.Void) def _PyComplex_AsCComplex(space, w_obj, result): """Return the Py_complex value of the complex number op. Modified: pypy/branch/fast-forward/pypy/module/cpyext/methodobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/methodobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/methodobject.py Fri Dec 3 17:56:20 2010 @@ -16,13 +16,14 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.objspace.std.tupleobject import W_TupleObject +PyCFunction_typedef = rffi.COpaquePtr('PyCFunction') PyCFunction = lltype.Ptr(lltype.FuncType([PyObject, PyObject], PyObject)) PyCFunctionKwArgs = lltype.Ptr(lltype.FuncType([PyObject, PyObject, PyObject], PyObject)) PyMethodDef = cpython_struct( 'PyMethodDef', [('ml_name', rffi.CCHARP), - ('ml_meth', PyCFunction), + ('ml_meth', PyCFunction_typedef), ('ml_flags', rffi.INT_real), ('ml_doc', rffi.CCHARP), ]) @@ -70,12 +71,14 @@ if space.is_true(w_kw) and not flags & METH_KEYWORDS: raise OperationError(space.w_TypeError, space.wrap( rffi.charp2str(self.ml.c_ml_name) + "() takes no keyword arguments")) + + func = rffi.cast(PyCFunction, self.ml.c_ml_meth) if flags & METH_KEYWORDS: func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) return generic_cpy_call(space, func, w_self, w_args, w_kw) elif flags & METH_NOARGS: if len(w_args.wrappeditems) == 0: - return generic_cpy_call(space, self.ml.c_ml_meth, w_self, None) + return generic_cpy_call(space, func, w_self, None) raise OperationError(space.w_TypeError, space.wrap( rffi.charp2str(self.ml.c_ml_name) + "() takes no arguments")) elif flags & METH_O: @@ -86,9 +89,9 @@ rffi.charp2str(self.ml.c_ml_name), len(w_args.wrappeditems)))) w_arg = w_args.wrappeditems[0] - return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) + return generic_cpy_call(space, func, w_self, w_arg) elif flags & METH_VARARGS: - return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_args) + return generic_cpy_call(space, func, w_self, w_args) else: # METH_OLDARGS, the really old style size = len(w_args.wrappeditems) if size == 1: @@ -97,7 +100,7 @@ w_arg = None else: w_arg = w_args - return generic_cpy_call(space, self.ml.c_ml_meth, w_self, w_arg) + return generic_cpy_call(space, func, w_self, w_arg) def get_doc(space, self): doc = self.ml.c_ml_doc @@ -231,6 +234,11 @@ def PyCFunction_NewEx(space, ml, w_self, w_name): return space.wrap(W_PyCFunctionObject(space, ml, w_self, w_name)) + at cpython_api([PyObject], PyCFunction_typedef) +def PyCFunction_GetFunction(space, w_obj): + cfunction = space.interp_w(W_PyCFunctionObject, w_obj) + return cfunction.ml.c_ml_meth + @cpython_api([PyObject], PyObject) def PyStaticMethod_New(space, w_func): return space.wrap(StaticMethod(w_func)) Modified: pypy/branch/fast-forward/pypy/module/cpyext/modsupport.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/modsupport.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/modsupport.py Fri Dec 3 17:56:20 2010 @@ -5,7 +5,7 @@ from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, - PyMethodDef, PyCFunction, PyStaticMethod_New) + PyMethodDef, PyStaticMethod_New) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_methodobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_methodobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_methodobject.py Fri Dec 3 17:56:20 2010 @@ -3,7 +3,8 @@ from pypy.module.cpyext.methodobject import PyMethodDef from pypy.module.cpyext.api import ApiFunction from pypy.module.cpyext.pyobject import PyObject, make_ref, Py_DecRef -from pypy.module.cpyext.methodobject import PyDescr_NewMethod +from pypy.module.cpyext.methodobject import ( + PyDescr_NewMethod, PyCFunction_typedef) from pypy.rpython.lltypesystem import rffi, lltype class AppTestMethodObject(AppTestCpythonExtensionBase): @@ -50,6 +51,16 @@ } ''' ), + ('isSameFunction', 'METH_O', + ''' + PyCFunction ptr = PyCFunction_GetFunction(args); + if (!ptr) return NULL; + if (ptr == foo_getarg_O) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; + ''' + ), ]) assert mod.getarg_O(1) == 1 raises(TypeError, mod.getarg_O) @@ -64,6 +75,8 @@ assert mod.getarg_OLD(1, 2) == (1, 2) assert mod.isCFunction(mod.getarg_O) == "getarg_O" + assert mod.isSameFunction(mod.getarg_O) + raises(TypeError, mod.isSameFunction, 1) class TestPyCMethodObject(BaseApiTest): def test_repr(self, space): @@ -78,7 +91,8 @@ ml = lltype.malloc(PyMethodDef, flavor='raw', zero=True) namebuf = rffi.str2charp('func') ml.c_ml_name = namebuf - ml.c_ml_meth = c_func.get_llhelper(space) + ml.c_ml_meth = rffi.cast(PyCFunction_typedef, + c_func.get_llhelper(space)) method = PyDescr_NewMethod(space, space.w_str, ml) assert repr(method).startswith( Modified: pypy/branch/fast-forward/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/typeobject.py Fri Dec 3 17:56:20 2010 @@ -16,10 +16,10 @@ track_reference, RefcountState, borrow_from) from pypy.interpreter.module import Module from pypy.module.cpyext import structmemberdefs -from pypy.module.cpyext.modsupport import convert_method_defs, PyCFunction +from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.state import State from pypy.module.cpyext.methodobject import ( - PyDescr_NewWrapper, PyCFunction_NewEx) + PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef) from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef, _Py_Dealloc from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( @@ -208,7 +208,7 @@ def setup_new_method_def(space): ptr = get_new_method_def(space) - ptr.c_ml_meth = rffi.cast(PyCFunction, + ptr.c_ml_meth = rffi.cast(PyCFunction_typedef, llhelper(tp_new_wrapper.api_func.functype, tp_new_wrapper.api_func.get_wrapper(space))) Modified: pypy/branch/fast-forward/pypy/module/cpyext/typeobjectdefs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/typeobjectdefs.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/typeobjectdefs.py Fri Dec 3 17:56:20 2010 @@ -8,7 +8,6 @@ from pypy.module.cpyext.modsupport import PyMethodDef -PyCFunction = Ptr(FuncType([PyObject, PyObject], PyObject)) P, FT, PyO = Ptr, FuncType, PyObject PyOPtr = Ptr(lltype.Array(PyO, hints={'nolength': True})) From afa at codespeak.net Fri Dec 3 17:58:58 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 17:58:58 +0100 (CET) Subject: [pypy-svn] r79786 - pypy/branch/fast-forward/pypy/module/cpyext Message-ID: <20101203165858.9DADE282BE0@codespeak.net> Author: afa Date: Fri Dec 3 17:58:57 2010 New Revision: 79786 Modified: pypy/branch/fast-forward/pypy/module/cpyext/cdatetime.py pypy/branch/fast-forward/pypy/module/cpyext/object.py pypy/branch/fast-forward/pypy/module/cpyext/unicodeobject.py Log: "error=nullptr" is now the default for pointer results Modified: pypy/branch/fast-forward/pypy/module/cpyext/cdatetime.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/cdatetime.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/cdatetime.py Fri Dec 3 17:58:57 2010 @@ -18,8 +18,7 @@ ('DeltaType', PyTypeObjectPtr), )) - at cpython_api([], lltype.Ptr(PyDateTime_CAPI), - error=lltype.nullptr(PyDateTime_CAPI)) + at cpython_api([], lltype.Ptr(PyDateTime_CAPI)) def _PyDateTime_Import(space): datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw', track_allocation=False) Modified: pypy/branch/fast-forward/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/object.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/object.py Fri Dec 3 17:58:57 2010 @@ -14,7 +14,7 @@ import pypy.module.__builtin__.operation as operation - at cpython_api([Py_ssize_t], rffi.VOIDP, error=lltype.nullptr(rffi.VOIDP.TO)) + at cpython_api([Py_ssize_t], rffi.VOIDP) def PyObject_MALLOC(space, size): return lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) Modified: pypy/branch/fast-forward/pypy/module/cpyext/unicodeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/unicodeobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/unicodeobject.py Fri Dec 3 17:58:57 2010 @@ -122,7 +122,7 @@ ref_unicode.c_buffer = rffi.unicode2wcharp(u) return ref_unicode.c_buffer - at cpython_api([PyObject], rffi.CWCHARP, error=lltype.nullptr(rffi.CWCHARP.TO)) + at cpython_api([PyObject], rffi.CWCHARP) def PyUnicode_AsUnicode(space, ref): """Return a read-only pointer to the Unicode object's internal Py_UNICODE buffer, NULL if unicode is not a Unicode object.""" From afa at codespeak.net Fri Dec 3 18:08:29 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 18:08:29 +0100 (CET) Subject: [pypy-svn] r79787 - pypy/branch/fast-forward/pypy/module/cpyext Message-ID: <20101203170829.8B05B282BE7@codespeak.net> Author: afa Date: Fri Dec 3 18:08:27 2010 New Revision: 79787 Modified: pypy/branch/fast-forward/pypy/module/cpyext/stubs.py Log: Remove from stubs.py already implemented functions Modified: pypy/branch/fast-forward/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/stubs.py Fri Dec 3 18:08:27 2010 @@ -4,73 +4,34 @@ from pypy.rpython.lltypesystem import rffi, lltype # we don't really care -PyTypeObjectPtr = lltype.Void -Py_ssize_t = lltype.Void -PyMethodDef = lltype.Void -PyGetSetDef = lltype.Void -PyMemberDef = lltype.Void -Py_buffer = lltype.Void -Py_complex = lltype.Void -va_list = lltype.Void -PyDateTime_Date = lltype.Void -PyDateTime_DateTime = lltype.Void -PyDateTime_Time = lltype.Void -wrapperbase = lltype.Void -FILE = lltype.Void -PyFileObject = lltype.Void -PyCodeObject = lltype.Void -PyFrameObject = lltype.Void -_inittab = lltype.Void -PyThreadState = lltype.Void -PyInterpreterState = lltype.Void -#PyOS_sighandler_t = lltype.Void -Py_UNICODE = lltype.Void -PyCompilerFlags = lltype.Void -_node = lltype.Void +PyTypeObjectPtr = rffi.VOIDP +Py_ssize_t = rffi.VOIDP +PyMethodDef = rffi.VOIDP +PyGetSetDef = rffi.VOIDP +PyMemberDef = rffi.VOIDP +Py_buffer = rffi.VOIDP +Py_complex = rffi.VOIDP +va_list = rffi.VOIDP +PyDateTime_Date = rffi.VOIDP +PyDateTime_DateTime = rffi.VOIDP +PyDateTime_Time = rffi.VOIDP +wrapperbase = rffi.VOIDP +FILE = rffi.VOIDP +PyFileObject = rffi.VOIDP +PyCodeObject = rffi.VOIDP +PyFrameObject = rffi.VOIDP +_inittab = rffi.VOIDP +PyThreadState = rffi.VOIDP +PyInterpreterState = rffi.VOIDP +PyOS_sighandler_t = rffi.VOIDP +Py_UNICODE = rffi.VOIDP +PyCompilerFlags = rffi.VOIDP +_node = rffi.VOIDP @cpython_api([PyObject], lltype.Void) def _PyObject_Del(space, op): raise NotImplementedError - at cpython_api([rffi.CCHARP, PyMethodDef], PyObject) -def Py_InitModule(space, name, methods): - """Create a new module object based on a name and table of functions, - returning the new module object. - - Older versions of Python did not support NULL as the value for the - methods argument.""" - borrow_from() - raise NotImplementedError - - at cpython_api([rffi.CCHARP, PyMethodDef, rffi.CCHARP], PyObject) -def Py_InitModule3(space, name, methods, doc): - """Create a new module object based on a name and table of functions, - returning the new module object. If doc is non-NULL, it will be used - to define the docstring for the module. - - Older versions of Python did not support NULL as the value for the - methods argument.""" - borrow_from() - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP, va_list], rffi.INT_real, error=0) -def PyArg_VaParse(space, args, format, vargs): - """Identical to PyArg_ParseTuple(), except that it accepts a va_list - rather than a variable number of arguments.""" - raise NotImplementedError - - at cpython_api([PyObject, PyObject, rffi.CCHARP, rffi.CCHARP, va_list], rffi.INT_real, error=0) -def PyArg_VaParseTupleAndKeywords(space, args, kw, format, keywords, vargs): - """Identical to PyArg_ParseTupleAndKeywords(), except that it accepts a - va_list rather than a variable number of arguments.""" - raise NotImplementedError - - at cpython_api([rffi.CCHARP, va_list], PyObject) -def Py_VaBuildValue(space, format, vargs): - """Identical to Py_BuildValue(), except that it accepts a va_list - rather than a variable number of arguments.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=0) def PyObject_CheckBuffer(space, obj): """Return 1 if obj supports the buffer interface otherwise 0.""" @@ -261,11 +222,6 @@ length. Return 0 on success and -1 (with raising an error) on error.""" raise NotImplementedError - at cpython_api([PyObject], PyObject) -def PyMemoryView_FromObject(space, obj): - """Return a memoryview object from an object that defines the buffer interface.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyByteArray_Check(space, o): """Return true if the object o is a bytearray object or an instance of a @@ -390,13 +346,6 @@ version since the definition of the bytecode changes often.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) -def PyCode_NewEmpty(space, filename, funcname, firstlineno): - """Return a new empty code object with the specified filename, - function name, and first line number. It is illegal to - exec or eval() the resulting code object.""" - raise NotImplementedError - @cpython_api([Py_complex, Py_complex], Py_complex) def _Py_c_sum(space, left, right): """Return the sum of two complex numbers, using the C Py_complex @@ -714,13 +663,6 @@ described there.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.INT_real], rffi.INT_real, error=-1) -def PyErr_WarnPy3k(space, message, stacklevel): - """Issue a DeprecationWarning with the given message and stacklevel - if the Py_Py3kWarningFlag flag is enabled. - """ - raise NotImplementedError - @cpython_api([], lltype.Void) def PyErr_SetInterrupt(space, ): """ @@ -746,14 +688,6 @@ only be called from the main thread.""" raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARP, PyObject, PyObject], PyObject) -def PyErr_NewExceptionWithDoc(space, name, doc, base, dict): - """Same as PyErr_NewException(), except that the new exception class can - easily be given a docstring: If doc is non-NULL, it will be used as the - docstring for the exception class. - """ - raise NotImplementedError - @cpython_api([PyObject], lltype.Void) def PyErr_WriteUnraisable(space, obj): """This utility function prints a warning message to sys.stderr when an @@ -1053,22 +987,6 @@ extension modules.""" raise NotImplementedError - at cpython_api([PyObject], lltype.Void) -def Py_VISIT(space, o): - """Call the visit callback, with arguments o and arg. If visit returns - a non-zero value, then return it. Using this macro, tp_traverse - handlers look like: - - static int - my_traverse(Noddy *self, visitproc visit, void *arg) - { - Py_VISIT(self->foo); - Py_VISIT(self->bar); - return 0; - } - """ - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyGen_Check(space, gen): """Return true if ob is a generator object; ob must not be NULL.""" @@ -1087,18 +1005,6 @@ NULL.""" raise NotImplementedError - at cpython_api([rffi.CCHARP], PyObject) -def PyImport_ImportModuleNoBlock(space, name): - """This version of PyImport_ImportModule() does not block. It's intended - to be used in C functions that import other modules to execute a function. - The import may block if another thread holds the import lock. The function - PyImport_ImportModuleNoBlock() never blocks. It first tries to fetch - the module from sys.modules and falls back to PyImport_ImportModule() - unless the lock is held, in which case the function will raise an - ImportError. - """ - raise NotImplementedError - @cpython_api([rffi.CCHARP, PyObject, PyObject, PyObject], PyObject) def PyImport_ImportModuleEx(space, name, globals, locals, fromlist): """ @@ -1879,29 +1785,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], PyObject) -def PyList_GET_ITEM(space, list, i): - """Macro form of PyList_GetItem() without error checking. - - This macro used an int for i. This might require changes in - your code for properly supporting 64-bit systems.""" - borrow_from() - raise NotImplementedError - - at cpython_api([PyObject, Py_ssize_t, PyObject], lltype.Void) -def PyList_SET_ITEM(space, list, i, o): - """Macro form of PyList_SetItem() without error checking. This is - normally only used to fill in new lists where there is no previous content. - - This macro "steals" a reference to item, and, unlike - PyList_SetItem(), does not discard a reference to any item that - it being replaced; any reference in list at position i will be - leaked. - - This macro used an int for i. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) def PyList_GetSlice(space, list, low, high): """Return a list of the objects in list containing the objects between low @@ -1941,30 +1824,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, rffi.INTP], lltype.Signed, error=-1) -def PyLong_AsLongAndOverflow(space, pylong, overflow): - """Return a C long representation of the contents of - pylong. If pylong is greater than LONG_MAX or less - than LONG_MIN, set *overflow to 1 or -1, - respectively, and return -1; otherwise, set *overflow to - 0. If any other exception occurs (for example a TypeError or - MemoryError), then -1 will be returned and *overflow will - be 0. - """ - raise NotImplementedError - - at cpython_api([PyObject, rffi.INTP], rffi.LONGLONG, error=-1) -def PyLong_AsLongLongAndOverflow(space, pylong, overflow): - """Return a C long long representation of the contents of - pylong. If pylong is greater than PY_LLONG_MAX or less - than PY_LLONG_MIN, set *overflow to 1 or -1, - respectively, and return -1; otherwise, set *overflow to - 0. If any other exception occurs (for example a TypeError or - MemoryError), then -1 will be returned and *overflow will - be 0. - """ - raise NotImplementedError - @cpython_api([PyObject], Py_ssize_t) def PyLong_AsSsize_t(space, pylong): """ @@ -1977,13 +1836,6 @@ """ raise NotImplementedError - at cpython_api([PyObject], rffi.ULONGLONG, error=-1) -def PyLong_AsUnsignedLongLongMask(space, io): - """Return a C unsigned long long from a Python long integer, without - checking for overflow. - """ - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], rffi.INT_real, error=-1) def PyMapping_DelItemString(space, o, key): """Remove the mapping for object key from the object o. Return -1 on @@ -2415,20 +2267,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject], Py_ssize_t) -def PyString_GET_SIZE(space, string): - """Macro form of PyString_Size() but without error checking. - - This macro returned an int type. This might require changes in - your code for properly supporting 64-bit systems.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL) -def PyString_AS_STRING(space, string): - """Macro form of PyString_AsString() but without error checking. Only - string objects are supported; no Unicode objects should be passed.""" - raise NotImplementedError - @cpython_api([PyObjectP], lltype.Void) def PyString_InternInPlace(space, string): """Intern the argument *string in place. The argument must be the address of a @@ -2607,15 +2445,6 @@ the cleanup function, no Python APIs should be called by func.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t], PyObject) -def PyTuple_GET_ITEM(space, p, pos): - """Like PyTuple_GetItem(), but does no checking of its arguments. - - This function used an int type for pos. This might require - changes in your code for properly supporting 64-bit systems.""" - borrow_from() - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) def PyTuple_GetSlice(space, p, low, high): """Take a slice of the tuple pointed to by p from low to high and return it @@ -2625,17 +2454,6 @@ require changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, PyObject], lltype.Void) -def PyTuple_SET_ITEM(space, p, pos, o): - """Like PyTuple_SetItem(), but does no error checking, and should only be - used to fill in brand new tuples. - - This function "steals" a reference to o. - - This function used an int type for pos. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def PyTuple_ClearFreeList(space, ): """Clear the free list. Return the total number of freed items. @@ -2656,12 +2474,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], rffi.INT_real, error=CANNOT_FAIL) -def PyType_HasFeature(space, o, feature): - """Return true if the type object o sets the feature feature. Type features - are denoted by single bit flags.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyType_IS_GC(space, o): """Return true if the type object includes support for the cycle detector; this From david at codespeak.net Fri Dec 3 18:15:17 2010 From: david at codespeak.net (david at codespeak.net) Date: Fri, 3 Dec 2010 18:15:17 +0100 (CET) Subject: [pypy-svn] r79788 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . test Message-ID: <20101203171517.74B7E282BE7@codespeak.net> Author: david Date: Fri Dec 3 18:15:15 2010 New Revision: 79788 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py Log: Fix for int_sub and refactor guard_value Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Fri Dec 3 18:15:15 2010 @@ -69,17 +69,17 @@ imm_a0 = self._check_imm_arg(a0) imm_a1 = self._check_imm_arg(a1) if not imm_a0 and imm_a1: - l0, box = self._ensure_value_is_boxed(a0, regalloc) + l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) l1 = regalloc.make_sure_var_in_reg(a1, [a0]) boxes.append(box) elif imm_a0 and not imm_a1: l0 = regalloc.make_sure_var_in_reg(a0) - l1, box = self._ensure_value_is_boxed(a1, regalloc) + l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) boxes.append(box) else: - l0, box = self._ensure_value_is_boxed(a0, regalloc) + l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) boxes.append(box) - l1, box = self._ensure_value_is_boxed(a1, regalloc) + l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) boxes.append(box) res = regalloc.force_allocate_reg(op.result, boxes) if l0.is_imm(): @@ -184,8 +184,7 @@ self.mc.MVN_rr(res.value, reg.value) regalloc.possibly_free_vars_for_op(op) - if op.result: - regalloc.possibly_free_var(op.result) + regalloc.possibly_free_var(op.result) return fcond #XXX check for a better way of doing this @@ -239,16 +238,23 @@ return self._emit_guard(op, regalloc, c.EQ) def emit_op_guard_value(self, op, regalloc, fcond): - a0 = op.getarg(0) - a1 = op.getarg(1) + boxes = list(op.getarglist()) + a0, a1 = boxes imm_a1 = self._check_imm_arg(a1) - l0 = regalloc.make_sure_var_in_reg(a0, imm_fine=False) - l1 = regalloc.make_sure_var_in_reg(a1, imm_fine=imm_a1) + l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) + boxes.append(box) + if not imm_a1: + l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(box) + else: + l1 = regalloc.make_sure_var_in_reg(a1) + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) + if l1.is_imm(): self.mc.CMP_ri(l0.value, l1.getint()) else: self.mc.CMP_rr(l0.value, l1.value) - regalloc.possibly_free_vars_for_op(op) return self._emit_guard(op, regalloc, c.EQ) emit_op_guard_nonnull = emit_op_guard_true Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py Fri Dec 3 18:15:15 2010 @@ -383,6 +383,67 @@ assert op.identifier == 2 assert cpu.get_latest_value_int(0) == 16 assert cpu.get_latest_value_int(1) == -1 + + def test_wrong_guard3(self): + # random seed: 8029 + # block length: 10 + faildescr1 = BasicFailDescr(1) + faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) + faildescr4 = BasicFailDescr(4) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + v16 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.UINT_LT, [ConstInt(-11), v7], v11), + ResOperation(rop.INT_GE, [v3, v5], v12), + ResOperation(rop.INT_INVERT, [v9], v13), + ResOperation(rop.GUARD_VALUE, [v13, ConstInt(14)], None, descr=faildescr3), + ResOperation(rop.INT_IS_ZERO, [v12], v14), + ResOperation(rop.INT_SUB, [v2, v13], v15), + ResOperation(rop.GUARD_VALUE, [v15, ConstInt(-32)], None, descr=faildescr4), + ResOperation(rop.INT_FLOORDIV, [v3, ConstInt(805306366)], v16), + ResOperation(rop.GUARD_VALUE, [v15, ConstInt(0)], None, descr=faildescr1), + ResOperation(rop.FINISH, [v10, v8, v1, v6, v4], None, descr=faildescr2), + ] + operations[3].setfailargs([]) + operations[-4].setfailargs([v15]) + operations[-2].setfailargs([v9, v4, v10, v11, v14]) + looptoken = LoopToken() + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, -39) + cpu.set_future_value_int(1, -18) + cpu.set_future_value_int(2, 1588243114) + cpu.set_future_value_int(3, -9) + cpu.set_future_value_int(4, -4) + cpu.set_future_value_int(5, 1252698794) + cpu.set_future_value_int(6, 0) + cpu.set_future_value_int(7, 715827882) + cpu.set_future_value_int(8, -15) + cpu.set_future_value_int(9, 536870912) + op = cpu.execute_token(looptoken) + assert op.identifier == 1 + assert cpu.get_latest_value_int(0) == -15 + assert cpu.get_latest_value_int(1) == -9 + assert cpu.get_latest_value_int(2) == 536870912 + assert cpu.get_latest_value_int(3) == 0 + assert cpu.get_latest_value_int(4) == 0 + def test_wrong_result(self): # generated by: # ../test/ test/test_zll_random.py -l -k arm -s --block-length=10 --random-seed=7389 From arigo at codespeak.net Fri Dec 3 18:50:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 3 Dec 2010 18:50:47 +0100 (CET) Subject: [pypy-svn] r79789 - in pypy/trunk/pypy/module/cpyext: . include test Message-ID: <20101203175047.7D4A3282BE3@codespeak.net> Author: arigo Date: Fri Dec 3 18:50:42 2010 New Revision: 79789 Modified: pypy/trunk/pypy/module/cpyext/api.py pypy/trunk/pypy/module/cpyext/include/tupleobject.h pypy/trunk/pypy/module/cpyext/slotdefs.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py pypy/trunk/pypy/module/cpyext/test/test_tupleobject.py pypy/trunk/pypy/module/cpyext/tupleobject.py Log: Partially revert r79688. More precisely, revert the changes to cpyext that were done in the rlist-jit branch. After much searching it seems to be the cause of the random crashes we systematically get nowadays trying to run "pypy translate.py". Without it, it seems to work fine again. Modified: pypy/trunk/pypy/module/cpyext/api.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/api.py (original) +++ pypy/trunk/pypy/module/cpyext/api.py Fri Dec 3 18:50:42 2010 @@ -369,7 +369,7 @@ }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) - for cpyname in 'Method List Int Long Dict Class'.split(): + for cpyname in 'Method List Int Long Dict Tuple Class'.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } ' 'Py%sObject' % (cpyname, )) build_exported_objects() Modified: pypy/trunk/pypy/module/cpyext/include/tupleobject.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/tupleobject.h (original) +++ pypy/trunk/pypy/module/cpyext/include/tupleobject.h Fri Dec 3 18:50:42 2010 @@ -10,19 +10,9 @@ /* defined in varargswrapper.c */ PyObject * PyTuple_Pack(Py_ssize_t, ...); -typedef struct { - PyObject_HEAD - PyObject **items; - Py_ssize_t size; -} PyTupleObject; +#define PyTuple_SET_ITEM PyTuple_SetItem +#define PyTuple_GET_ITEM PyTuple_GetItem -#define PyTuple_GET_ITEM PyTuple_GetItem - -/* Macro, trading safety for speed */ -#define PyTuple_GET_SIZE(op) (((PyTupleObject *)(op))->size) - -/* Macro, *only* to be used to fill in brand new tuples */ -#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->items[i] = v) #ifdef __cplusplus } Modified: pypy/trunk/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/slotdefs.py (original) +++ pypy/trunk/pypy/module/cpyext/slotdefs.py Fri Dec 3 18:50:42 2010 @@ -28,14 +28,14 @@ def check_num_args(space, ob, n): from pypy.module.cpyext.tupleobject import PyTuple_CheckExact, \ - _PyTuple_Size_Fast + PyTuple_GET_SIZE if not PyTuple_CheckExact(space, ob): raise OperationError(space.w_SystemError, space.wrap("PyArg_UnpackTuple() argument list is not a tuple")) - if n == _PyTuple_Size_Fast(space, ob): + if n == PyTuple_GET_SIZE(space, ob): return raise operationerrfmt(space.w_TypeError, - "expected %d arguments, got %d", n, _PyTuple_Size_Fast(space, ob)) + "expected %d arguments, got %d", n, PyTuple_GET_SIZE(space, ob)) def wrap_init(space, w_self, w_args, func, w_kwargs): func_init = rffi.cast(initproc, func) Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Fri Dec 3 18:50:42 2010 @@ -545,17 +545,16 @@ PyObject *true = Py_True; PyObject *tup = NULL; int refcnt = true->ob_refcnt; - int refcnt_middle, refcnt_after; + int refcnt_after; tup = PyTuple_New(1); Py_INCREF(true); if (PyTuple_SetItem(tup, 0, true) < 0) return NULL; - refcnt_middle = true->ob_refcnt; - Py_DECREF(tup); refcnt_after = true->ob_refcnt; - fprintf(stderr, "REFCNT2 %i %i %i\\n", refcnt, refcnt_middle, refcnt_after); - return PyBool_FromLong(refcnt_after == refcnt && refcnt_middle == refcnt+1); + Py_DECREF(tup); + fprintf(stderr, "REFCNT2 %i %i\\n", refcnt, refcnt_after); + return PyBool_FromLong(refcnt_after == refcnt); } static PyMethodDef methods[] = { Modified: pypy/trunk/pypy/module/cpyext/test/test_tupleobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_tupleobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_tupleobject.py Fri Dec 3 18:50:42 2010 @@ -7,34 +7,24 @@ class TestTupleObject(BaseApiTest): def test_tupleobject(self, space, api): assert not api.PyTuple_Check(space.w_None) - #assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 XXX + assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 atuple = space.newtuple([0, 1, 'yay']) assert api.PyTuple_Size(atuple) == 3 - #raises(TypeError, api.PyTuple_Size(space.newlist([]))) XXX + assert api.PyTuple_GET_SIZE(atuple) == 3 + raises(TypeError, api.PyTuple_Size(space.newlist([]))) api.PyErr_Clear() def test_tuple_resize(self, space, api): - ref_tup = api.PyTuple_New(3) + py_tuple = api.PyTuple_New(3) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - ar[0] = rffi.cast(PyObject, ref_tup) + ar[0] = rffi.cast(PyObject, make_ref(space, py_tuple)) api._PyTuple_Resize(ar, 2) - assert ar[0] == rffi.cast(PyObject, ref_tup) - # ^^^ our _PyTuple_Resize does not actually need to change the ptr so far - assert api.PyTuple_Size(ar[0]) == 2 + py_tuple = from_ref(space, ar[0]) + assert len(py_tuple.wrappeditems) == 2 api._PyTuple_Resize(ar, 10) - assert api.PyTuple_Size(ar[0]) == 10 + py_tuple = from_ref(space, ar[0]) + assert len(py_tuple.wrappeditems) == 10 api.Py_DecRef(ar[0]) lltype.free(ar, flavor='raw') - - def test_tuple_setup(self, space, api): - ref_tup = api.PyTuple_New(2) - ref0 = make_ref(space, space.wrap(123)) - api.PyTuple_SetItem(ref_tup, 0, ref0) - ref1 = make_ref(space, space.wrap(456)) - api.PyTuple_SetItem(ref_tup, 1, ref1) - - w_tup = from_ref(space, ref_tup) - assert space.is_true(space.eq(w_tup, space.wrap((123, 456)))) - api.Py_DecRef(ref_tup) Modified: pypy/trunk/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/tupleobject.py (original) +++ pypy/trunk/pypy/module/cpyext/tupleobject.py Fri Dec 3 18:50:42 2010 @@ -1,144 +1,55 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers, PyObjectFields, - cpython_struct, bootstrap_function) + build_type_checkers) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - borrow_from, make_ref, from_ref, make_typedescr, get_typedescr, Reference, - track_reference) + borrow_from, make_ref, from_ref) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall +from pypy.objspace.std.tupleobject import W_TupleObject -## -## Implementation of PyTupleObject -## =============================== -## -## We have the same problem as PyStringObject: a PyTupleObject can be -## initially used in a read-write way with PyTuple_New(), PyTuple_SetItem() -## and _PyTuple_Resize(). -## -## The 'size' and 'items' fields of a PyTupleObject are always valid. -## Apart from that detail, see the big comment in stringobject.py for -## more information. -## - -ARRAY_OF_PYOBJ = rffi.CArrayPtr(PyObject) -PyTupleObjectStruct = lltype.ForwardReference() -PyTupleObject = lltype.Ptr(PyTupleObjectStruct) -PyTupleObjectFields = PyObjectFields + \ - (("items", ARRAY_OF_PYOBJ), ("size", Py_ssize_t)) -cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) - - at bootstrap_function -def init_tupleobject(space): - "Type description of PyTupleObject" - make_typedescr(space.w_tuple.instancetypedef, - basestruct=PyTupleObject.TO, - attach=tuple_attach, - dealloc=tuple_dealloc, - realize=tuple_realize) PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple") -def new_empty_tuple(space, length): - """ - Allocate a PyTupleObject and its array, but without a corresponding - interpreter object. The array items may be mutated, until - tuple_realize() is called. - """ - typedescr = get_typedescr(space.w_tuple.instancetypedef) - py_obj = typedescr.allocate(space, space.w_tuple) - py_tup = rffi.cast(PyTupleObject, py_obj) - - py_tup.c_items = lltype.malloc(ARRAY_OF_PYOBJ.TO, length, - flavor='raw', zero=True) - py_tup.c_size = length - return py_tup - -def tuple_attach(space, py_obj, w_obj): - """ - Fills a newly allocated PyTupleObject with the given tuple object. - """ - items_w = space.fixedview(w_obj) - py_tup = rffi.cast(PyTupleObject, py_obj) - py_tup.c_items = lltype.nullptr(ARRAY_OF_PYOBJ.TO) - py_tup.c_size = len(items_w) - -def tuple_realize(space, py_obj): - """ - Creates the tuple in the interpreter. The PyTupleObject items array - must not be modified after this call. - """ - py_tup = rffi.cast(PyTupleObject, py_obj) - # If your CPython extension creates a self-referential tuple - # with PyTuple_SetItem(), you loose. - c_items = py_tup.c_items - items_w = [from_ref(space, c_items[i]) for i in range(py_tup.c_size)] - w_obj = space.newtuple(items_w) - track_reference(space, py_obj, w_obj) - return w_obj - - at cpython_api([PyObject], lltype.Void, external=False) -def tuple_dealloc(space, py_obj): - """Frees allocated PyTupleObject resources. - """ - py_tup = rffi.cast(PyTupleObject, py_obj) - if py_tup.c_items: - for i in range(py_tup.c_size): - Py_DecRef(space, py_tup.c_items[i]) - lltype.free(py_tup.c_items, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) - -#_______________________________________________________________________ - @cpython_api([Py_ssize_t], PyObject) def PyTuple_New(space, size): - return rffi.cast(PyObject, new_empty_tuple(space, size)) + return space.newtuple([space.w_None] * size) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PyTuple_SetItem(space, ref, pos, ref_item): - # XXX steals a reference at the level of PyObjects. Don't try to - # XXX call this function with an interpreter object as ref_item! - - # XXX do PyTuple_Check, without forcing ref as an interpreter object - # XXX -- then if it fails it should also steal a reference, test it!!! - ref_tup = rffi.cast(PyTupleObject, ref) - if not ref_tup.c_items: - msg = "PyTuple_SetItem() called on an already-escaped tuple object" - raise OperationError(space.w_SystemError, space.wrap(msg)) - ref_old = ref_tup.c_items[pos] - ref_tup.c_items[pos] = ref_item # SetItem steals a reference! - Py_DecRef(space, ref_old) +def PyTuple_SetItem(space, w_t, pos, w_obj): + if not PyTuple_Check(space, w_t): + # XXX this should also steal a reference, test it!!! + PyErr_BadInternalCall(space) + assert isinstance(w_t, W_TupleObject) + w_t.wrappeditems[pos] = w_obj + Py_DecRef(space, w_obj) # SetItem steals a reference! return 0 @cpython_api([PyObject, Py_ssize_t], PyObject) -def PyTuple_GetItem(space, ref, pos): - # XXX do PyTuple_Check, without forcing ref as an interpreter object - ref_tup = rffi.cast(PyTupleObject, ref) - if ref_tup.c_items: - return Reference(ref_tup.c_items[pos]) # borrowed reference - else: - w_t = from_ref(space, ref) - w_obj = space.getitem(w_t, space.wrap(pos)) - return borrow_from(w_t, w_obj) - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def _PyTuple_Size_Fast(space, ref): - # custom version: it's not a macro, so it can be called from other .py - # files; but it doesn't include PyTuple_Check() - ref_tup = rffi.cast(PyTupleObject, ref) - return ref_tup.c_size +def PyTuple_GetItem(space, w_t, pos): + if not PyTuple_Check(space, w_t): + PyErr_BadInternalCall(space) + assert isinstance(w_t, W_TupleObject) + w_obj = w_t.wrappeditems[pos] + return borrow_from(w_t, w_obj) + + at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) +def PyTuple_GET_SIZE(space, w_t): + """Return the size of the tuple p, which must be non-NULL and point to a tuple; + no error checking is performed. """ + assert isinstance(w_t, W_TupleObject) + return len(w_t.wrappeditems) @cpython_api([PyObject], Py_ssize_t, error=-1) def PyTuple_Size(space, ref): """Take a pointer to a tuple object, and return the size of that tuple.""" - # XXX do PyTuple_Check, without forcing ref as an interpreter object - ref_tup = rffi.cast(PyTupleObject, ref) - return ref_tup.c_size + if not PyTuple_Check(space, ref): + raise OperationError(space.w_TypeError, + space.wrap("expected tuple object")) + return PyTuple_GET_SIZE(space, ref) @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyTuple_Resize(space, refp, newsize): +def _PyTuple_Resize(space, ref, newsize): """Can be used to resize a tuple. newsize will be the new length of the tuple. Because tuples are supposed to be immutable, this should only be used if there is only one reference to the object. Do not use this if the tuple may already @@ -149,22 +60,18 @@ this function. If the object referenced by *p is replaced, the original *p is destroyed. On failure, returns -1 and sets *p to NULL, and raises MemoryError or SystemError.""" - # XXX do PyTuple_Check, without forcing ref as an interpreter object - # XXX -- then if it fails it should reset refp[0] to null - ref_tup = rffi.cast(PyTupleObject, refp[0]) - c_newitems = lltype.malloc(ARRAY_OF_PYOBJ.TO, newsize, - flavor='raw', zero=True) - c_olditems = ref_tup.c_items - if not c_olditems: - msg = "_PyTuple_Resize() called on an already-escaped tuple object" - raise OperationError(space.w_SystemError, space.wrap(msg)) - oldsize = ref_tup.c_size - for i in range(min(oldsize, newsize)): - c_newitems[i] = c_olditems[i] - # decref items deleted by shrinkage - for i in range(newsize, oldsize): - Py_DecRef(space, c_olditems[i]) - ref_tup.c_items = c_newitems - ref_tup.c_size = newsize - lltype.free(c_olditems, flavor='raw') + py_tuple = from_ref(space, ref[0]) + if not PyTuple_Check(space, py_tuple): + PyErr_BadInternalCall(space) + assert isinstance(py_tuple, W_TupleObject) + py_newtuple = PyTuple_New(space, newsize) + + to_cp = newsize + oldsize = len(py_tuple.wrappeditems) + if oldsize < newsize: + to_cp = oldsize + for i in range(to_cp): + py_newtuple.wrappeditems[i] = py_tuple.wrappeditems[i] + Py_DecRef(space, ref[0]) + ref[0] = make_ref(space, py_newtuple) return 0 From afa at codespeak.net Fri Dec 3 18:54:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 18:54:43 +0100 (CET) Subject: [pypy-svn] r79790 - in pypy/branch/fast-forward/pypy/module/cpyext: . test Message-ID: <20101203175443.F21F3282BE3@codespeak.net> Author: afa Date: Fri Dec 3 18:54:41 2010 New Revision: 79790 Modified: pypy/branch/fast-forward/pypy/module/cpyext/floatobject.py pypy/branch/fast-forward/pypy/module/cpyext/intobject.py pypy/branch/fast-forward/pypy/module/cpyext/stubs.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_floatobject.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_intobject.py Log: PyFloat_FromString, PyInt_FromString Modified: pypy/branch/fast-forward/pypy/module/cpyext/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/floatobject.py Fri Dec 3 18:54:41 2010 @@ -1,6 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import (CANNOT_FAIL, cpython_api, PyObject, - build_type_checkers) +from pypy.module.cpyext.api import ( + CANNOT_FAIL, cpython_api, PyObject, build_type_checkers, CONST_STRING) from pypy.interpreter.error import OperationError PyFloat_Check, PyFloat_CheckExact = build_type_checkers("Float") @@ -24,4 +24,12 @@ """ Returns the o converted to a float object on success, or NULL on failure. This is the equivalent of the Python expression float(o).""" - return space.float(w_obj) + return space.call_function(space.w_float, w_obj) + + at cpython_api([PyObject, rffi.CCHARPP], PyObject) +def PyFloat_FromString(space, w_obj, _): + """Create a PyFloatObject object based on the string value in str, or + NULL on failure. The pend argument is ignored. It remains only for + backward compatibility.""" + return space.call_function(space.w_float, w_obj) + Modified: pypy/branch/fast-forward/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/intobject.py Fri Dec 3 18:54:41 2010 @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, - build_type_checkers, Py_ssize_t) +from pypy.module.cpyext.api import ( + cpython_api, build_type_checkers, PyObject, + CONST_STRING, CANNOT_FAIL, Py_ssize_t) from pypy.rlib.rarithmetic import r_uint PyInt_Check, PyInt_CheckExact = build_type_checkers("Int") @@ -63,3 +64,25 @@ """ return space.wrap(ival) # XXX this is wrong on win64 + at cpython_api([CONST_STRING, rffi.CCHARPP, rffi.INT_real], PyObject) +def PyInt_FromString(space, str, pend, base): + """Return a new PyIntObject or PyLongObject based on the string + value in str, which is interpreted according to the radix in base. If + pend is non-NULL, *pend will point to the first character in str which + follows the representation of the number. If base is 0, the radix will be + determined based on the leading characters of str: if str starts with + '0x' or '0X', radix 16 will be used; if str starts with '0', radix + 8 will be used; otherwise radix 10 will be used. If base is not 0, it + must be between 2 and 36, inclusive. Leading spaces are ignored. If + there are no digits, ValueError will be raised. If the string represents + a number too large to be contained within the machine's long int type + and overflow warnings are being suppressed, a PyLongObject will be + returned. If overflow warnings are not being suppressed, NULL will be + returned in this case.""" + s = rffi.charp2str(str) + w_str = space.wrap(s) + w_base = space.wrap(rffi.cast(lltype.Signed, base)) + if pend: + pend[0] = rffi.ptradd(str, len(s)) + return space.call_function(space.w_int, w_str, w_base) + Modified: pypy/branch/fast-forward/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/stubs.py Fri Dec 3 18:54:41 2010 @@ -842,13 +842,6 @@ failure; the appropriate exception will be set.""" raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARPP], PyObject) -def PyFloat_FromString(space, str, pend): - """Create a PyFloatObject object based on the string value in str, or - NULL on failure. The pend argument is ignored. It remains only for - backward compatibility.""" - raise NotImplementedError - @cpython_api([rffi.VOIDP_real], PyObject) def PyFloat_GetInfo(space, info): """Return a structseq instance which contains information about the @@ -1735,23 +1728,6 @@ """ raise NotImplementedError - at cpython_api([rffi.CCHARP, rffi.CCHARPP, rffi.INT_real], PyObject) -def PyInt_FromString(space, str, pend, base): - """Return a new PyIntObject or PyLongObject based on the string - value in str, which is interpreted according to the radix in base. If - pend is non-NULL, *pend will point to the first character in str which - follows the representation of the number. If base is 0, the radix will be - determined based on the leading characters of str: if str starts with - '0x' or '0X', radix 16 will be used; if str starts with '0', radix - 8 will be used; otherwise radix 10 will be used. If base is not 0, it - must be between 2 and 36, inclusive. Leading spaces are ignored. If - there are no digits, ValueError will be raised. If the string represents - a number too large to be contained within the machine's long int type - and overflow warnings are being suppressed, a PyLongObject will be - returned. If overflow warnings are not being suppressed, NULL will be - returned in this case.""" - raise NotImplementedError - @cpython_api([rffi.SIZE_T], PyObject) def PyInt_FromSize_t(space, ival): """Create a new integer object with a value of ival. If the value exceeds Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_floatobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_floatobject.py Fri Dec 3 18:54:41 2010 @@ -1,4 +1,5 @@ from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase class TestFloatObject(BaseApiTest): def test_floatobject(self, space, api): @@ -11,9 +12,24 @@ def test_coerce(self, space, api): assert space.type(api.PyNumber_Float(space.wrap(3))) is space.w_float + assert space.type(api.PyNumber_Float(space.wrap("3"))) is space.w_float class Coerce(object): def __float__(self): return 42.5 assert space.eq_w(api.PyNumber_Float(space.wrap(Coerce())), space.wrap(42.5)) + +class AppTestFloatObject(AppTestCpythonExtensionBase): + def test_fromstring(self): + module = self.import_extension('foo', [ + ("from_string", "METH_NOARGS", + """ + PyObject* str = PyString_FromString("1234.56"); + PyObject* res = PyFloat_FromString(str, NULL); + Py_DECREF(str); + return res; + """), + ]) + assert module.from_string() == 1234.56 + assert type(module.from_string()) is float Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_intobject.py Fri Dec 3 18:54:41 2010 @@ -1,4 +1,5 @@ from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase import sys class TestIntObject(BaseApiTest): @@ -34,3 +35,14 @@ def __int__(self): return 42 assert api.PyInt_AsLong(space.wrap(Coerce())) == 42 + +class AppTestIntObject(AppTestCpythonExtensionBase): + def test_fromstring(self): + module = self.import_extension('foo', [ + ("from_string", "METH_NOARGS", + """ + return PyInt_FromString("1234", NULL, 16); + """), + ]) + assert module.from_string() == 0x1234 + assert type(module.from_string()) is int From afa at codespeak.net Fri Dec 3 19:09:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 19:09:13 +0100 (CET) Subject: [pypy-svn] r79791 - in pypy/branch/fast-forward: . lib-python/modified-2.5.2/test lib_pypy/_ctypes pypy pypy/annotation pypy/annotation/test pypy/doc/config pypy/jit/backend/x86/test pypy/jit/metainterp/optimizeopt pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/test pypy/rlib pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/memory/gc pypy/rpython/memory/gc/test pypy/rpython/memory/gctransform pypy/rpython/ootypesystem pypy/rpython/tool pypy/tool Message-ID: <20101203180913.0945A282BE7@codespeak.net> Author: afa Date: Fri Dec 3 19:09:09 2010 New Revision: 79791 Added: pypy/branch/fast-forward/lib-python/modified-2.5.2/test/seq_tests.py - copied unchanged from r79790, pypy/trunk/lib-python/modified-2.5.2/test/seq_tests.py Modified: pypy/branch/fast-forward/ (props changed) pypy/branch/fast-forward/lib_pypy/_ctypes/array.py pypy/branch/fast-forward/lib_pypy/_ctypes/basics.py pypy/branch/fast-forward/lib_pypy/_ctypes/pointer.py pypy/branch/fast-forward/lib_pypy/_ctypes/primitive.py pypy/branch/fast-forward/pypy/ (props changed) pypy/branch/fast-forward/pypy/annotation/annrpython.py pypy/branch/fast-forward/pypy/annotation/binaryop.py pypy/branch/fast-forward/pypy/annotation/bookkeeper.py pypy/branch/fast-forward/pypy/annotation/description.py pypy/branch/fast-forward/pypy/annotation/model.py pypy/branch/fast-forward/pypy/annotation/specialize.py pypy/branch/fast-forward/pypy/annotation/test/test_annrpython.py pypy/branch/fast-forward/pypy/doc/config/objspace.usemodules.array.txt (props changed) pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_runner.py pypy/branch/fast-forward/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/fast-forward/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/fast-forward/pypy/module/cpyext/api.py pypy/branch/fast-forward/pypy/module/cpyext/include/tupleobject.h pypy/branch/fast-forward/pypy/module/cpyext/slotdefs.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_tupleobject.py pypy/branch/fast-forward/pypy/module/cpyext/tupleobject.py pypy/branch/fast-forward/pypy/rlib/rerased.py (contents, props changed) pypy/branch/fast-forward/pypy/rlib/test/test_rerased.py (props changed) pypy/branch/fast-forward/pypy/rpython/lltypesystem/rbuiltin.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/rclass.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/rdict.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/rpbc.py pypy/branch/fast-forward/pypy/rpython/memory/gc/minimark.py pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/fast-forward/pypy/rpython/memory/gctransform/boehm.py pypy/branch/fast-forward/pypy/rpython/memory/gctransform/framework.py pypy/branch/fast-forward/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/fast-forward/pypy/rpython/memory/gctransform/support.py pypy/branch/fast-forward/pypy/rpython/ootypesystem/rpbc.py pypy/branch/fast-forward/pypy/rpython/rpbc.py pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py pypy/branch/fast-forward/pypy/tool/error.py Log: Merge from trunk: -r79741:79790 Modified: pypy/branch/fast-forward/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/fast-forward/lib_pypy/_ctypes/array.py Fri Dec 3 19:09:09 2010 @@ -150,12 +150,9 @@ __metaclass__ = ArrayMeta _ffiargshape = 'P' - def __new__(cls, *args): - self = _CData.__new__(cls, *args) - self._buffer = self._ffiarray(self._length_, autofree=True) - return self - def __init__(self, *args): + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(self._length_, autofree=True) for i, arg in enumerate(args): self[i] = arg Modified: pypy/branch/fast-forward/lib_pypy/_ctypes/basics.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_ctypes/basics.py (original) +++ pypy/branch/fast-forward/lib_pypy/_ctypes/basics.py Fri Dec 3 19:09:09 2010 @@ -86,11 +86,14 @@ self._buffer = None def __repr__(self): - return repr(self._obj) + return '' % (self._obj,) def __eq__(self, other): return self._obj == other + def __ne__(self, other): + return self._obj != other + class _CData(object): """ The most basic object for all ctypes types """ Modified: pypy/branch/fast-forward/lib_pypy/_ctypes/pointer.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_ctypes/pointer.py (original) +++ pypy/branch/fast-forward/lib_pypy/_ctypes/pointer.py Fri Dec 3 19:09:09 2010 @@ -59,7 +59,8 @@ def set_type(self, TP): ffiarray = _rawffi.Array('P') def __init__(self, value=None): - self._buffer = ffiarray(1, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = ffiarray(1, autofree=True) if value is not None: self.contents = value self._ffiarray = ffiarray Modified: pypy/branch/fast-forward/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/fast-forward/lib_pypy/_ctypes/primitive.py Fri Dec 3 19:09:09 2010 @@ -288,12 +288,9 @@ __metaclass__ = SimpleType _type_ = 'i' - def __new__(cls, *args, **kwds): - self = _CData.__new__(cls, *args, **kwds) - self._buffer = self._ffiarray(1, autofree=True) - return self - def __init__(self, value=DEFAULT_VALUE): + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(1, autofree=True) if value is not DEFAULT_VALUE: self.value = value Modified: pypy/branch/fast-forward/pypy/annotation/annrpython.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/annrpython.py (original) +++ pypy/branch/fast-forward/pypy/annotation/annrpython.py Fri Dec 3 19:09:09 2010 @@ -145,7 +145,7 @@ classdef.add_source_for_attribute(attr, classdef.classdesc) self.bookkeeper assert isinstance(s_result, annmodel.SomePBC) - olddesc = s_result.descriptions.iterkeys().next() + olddesc = s_result.any_description() desc = olddesc.bind_self(classdef) args = self.bookkeeper.build_args("simple_call", args_s[:]) desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc], Modified: pypy/branch/fast-forward/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/binaryop.py (original) +++ pypy/branch/fast-forward/pypy/annotation/binaryop.py Fri Dec 3 19:09:09 2010 @@ -867,7 +867,7 @@ def getitem((p, obj)): assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - def setitem((p, obj)): + def setitem((p, obj), s_value): assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) class __extend__(pairtype(SomeObject, SomePtr)): Modified: pypy/branch/fast-forward/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/fast-forward/pypy/annotation/bookkeeper.py Fri Dec 3 19:09:09 2010 @@ -262,7 +262,7 @@ args_s, s_result) def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result): - descs = s_callable.descriptions.keys() + descs = list(s_callable.descriptions) if not descs: return family = descs[0].getcallfamily() @@ -590,7 +590,7 @@ assert s_attr.is_constant() attr = s_attr.const - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue @@ -633,7 +633,7 @@ """Analyse a call to a SomePBC() with the given args (list of annotations). """ - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue first = descs[0] Modified: pypy/branch/fast-forward/pypy/annotation/description.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/description.py (original) +++ pypy/branch/fast-forward/pypy/annotation/description.py Fri Dec 3 19:09:09 2010 @@ -672,7 +672,7 @@ if isinstance(s_init, SomePBC): assert len(s_init.descriptions) == 1, ( "unexpected dynamic __init__?") - initfuncdesc = s_init.descriptions.keys()[0] + initfuncdesc, = s_init.descriptions if isinstance(initfuncdesc, FunctionDesc): initmethdesc = bookkeeper.getmethoddesc(initfuncdesc, classdef, @@ -800,8 +800,8 @@ desc.selfclassdef, desc.name, commonflags) - del descs[desc] - descs[newdesc] = None + descs.remove(desc) + descs.add(newdesc) # --- case 1 --- groups = {} @@ -816,7 +816,7 @@ for desc2 in group: cdef2 = desc2.selfclassdef if cdef1 is not cdef2 and cdef1.issubclass(cdef2): - del descs[desc1] + descs.remove(desc1) break simplify_desc_set = staticmethod(simplify_desc_set) Modified: pypy/branch/fast-forward/pypy/annotation/model.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/model.py (original) +++ pypy/branch/fast-forward/pypy/annotation/model.py Fri Dec 3 19:09:09 2010 @@ -366,8 +366,8 @@ immutable = True def __init__(self, descriptions, can_be_None=False, subset_of=None): - # descriptions is a set of Desc instances. - descriptions = dict.fromkeys(descriptions) + # descriptions is a set of Desc instances + descriptions = set(descriptions) self.descriptions = descriptions self.can_be_None = can_be_None self.subset_of = subset_of @@ -389,6 +389,9 @@ if desc.pyobj is not None: self.const = desc.pyobj + def any_description(self): + return iter(self.descriptions).next() + def getKind(self): "Return the common Desc class of all descriptions in this PBC." kinds = {} Modified: pypy/branch/fast-forward/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/specialize.py (original) +++ pypy/branch/fast-forward/pypy/annotation/specialize.py Fri Dec 3 19:09:09 2010 @@ -345,7 +345,8 @@ key.append(s.const) elif isinstance(s, SomePBC) and len(s.descriptions) == 1: # for test_specialize_arg_bound_method - key.append(s.descriptions.keys()[0]) + desc, = s.descriptions + key.append(desc) else: raise Exception("specialize:arg(%d): argument not constant: %r" % (i, s)) Modified: pypy/branch/fast-forward/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/fast-forward/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/fast-forward/pypy/annotation/test/test_annrpython.py Fri Dec 3 19:09:09 2010 @@ -1010,7 +1010,7 @@ bookkeeper = a.bookkeeper def getmdesc(bmeth): - return bookkeeper.immutablevalue(bmeth).descriptions.keys()[0] + return bookkeeper.immutablevalue(bmeth).any_description() mdescA_m = getmdesc(A().m) mdescC_m = getmdesc(C().m) @@ -2862,7 +2862,7 @@ assert s.items[0].flags == {'access_directly': True} assert isinstance(s.items[1], annmodel.SomePBC) assert len(s.items[1].descriptions) == 1 - assert s.items[1].descriptions.keys()[0].flags == {'access_directly': + assert s.items[1].any_description().flags == {'access_directly': True} assert isinstance(s.items[2], annmodel.SomeInstance) assert s.items[2].flags == {'access_directly': True} Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_runner.py Fri Dec 3 19:09:09 2010 @@ -82,7 +82,7 @@ # relative addressing to work properly. addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.setup() + self.cpu.assembler.setup_once() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] Modified: pypy/branch/fast-forward/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/api.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/api.py Fri Dec 3 19:09:09 2010 @@ -374,7 +374,7 @@ }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) - for cpyname in 'Method List Int Long Dict Class'.split(): + for cpyname in 'Method List Int Long Dict Tuple Class'.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } ' 'Py%sObject' % (cpyname, )) build_exported_objects() Modified: pypy/branch/fast-forward/pypy/module/cpyext/include/tupleobject.h ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/include/tupleobject.h (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/include/tupleobject.h Fri Dec 3 19:09:09 2010 @@ -10,19 +10,9 @@ /* defined in varargswrapper.c */ PyObject * PyTuple_Pack(Py_ssize_t, ...); -typedef struct { - PyObject_HEAD - PyObject **items; - Py_ssize_t size; -} PyTupleObject; +#define PyTuple_SET_ITEM PyTuple_SetItem +#define PyTuple_GET_ITEM PyTuple_GetItem -#define PyTuple_GET_ITEM PyTuple_GetItem - -/* Macro, trading safety for speed */ -#define PyTuple_GET_SIZE(op) (((PyTupleObject *)(op))->size) - -/* Macro, *only* to be used to fill in brand new tuples */ -#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->items[i] = v) #ifdef __cplusplus } Modified: pypy/branch/fast-forward/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/slotdefs.py Fri Dec 3 19:09:09 2010 @@ -28,14 +28,14 @@ def check_num_args(space, ob, n): from pypy.module.cpyext.tupleobject import PyTuple_CheckExact, \ - _PyTuple_Size_Fast + PyTuple_GET_SIZE if not PyTuple_CheckExact(space, ob): raise OperationError(space.w_SystemError, space.wrap("PyArg_UnpackTuple() argument list is not a tuple")) - if n == _PyTuple_Size_Fast(space, ob): + if n == PyTuple_GET_SIZE(space, ob): return raise operationerrfmt(space.w_TypeError, - "expected %d arguments, got %d", n, _PyTuple_Size_Fast(space, ob)) + "expected %d arguments, got %d", n, PyTuple_GET_SIZE(space, ob)) def wrap_init(space, w_self, w_args, func, w_kwargs): func_init = rffi.cast(initproc, func) Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py Fri Dec 3 19:09:09 2010 @@ -554,17 +554,16 @@ PyObject *true = Py_True; PyObject *tup = NULL; int refcnt = true->ob_refcnt; - int refcnt_middle, refcnt_after; + int refcnt_after; tup = PyTuple_New(1); Py_INCREF(true); if (PyTuple_SetItem(tup, 0, true) < 0) return NULL; - refcnt_middle = true->ob_refcnt; - Py_DECREF(tup); refcnt_after = true->ob_refcnt; - fprintf(stderr, "REFCNT2 %i %i %i\\n", refcnt, refcnt_middle, refcnt_after); - return PyBool_FromLong(refcnt_after == refcnt && refcnt_middle == refcnt+1); + Py_DECREF(tup); + fprintf(stderr, "REFCNT2 %i %i\\n", refcnt, refcnt_after); + return PyBool_FromLong(refcnt_after == refcnt); } static PyMethodDef methods[] = { Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_tupleobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_tupleobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_tupleobject.py Fri Dec 3 19:09:09 2010 @@ -7,34 +7,24 @@ class TestTupleObject(BaseApiTest): def test_tupleobject(self, space, api): assert not api.PyTuple_Check(space.w_None) - #assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 XXX + assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 atuple = space.newtuple([0, 1, 'yay']) assert api.PyTuple_Size(atuple) == 3 - #raises(TypeError, api.PyTuple_Size(space.newlist([]))) XXX + assert api.PyTuple_GET_SIZE(atuple) == 3 + raises(TypeError, api.PyTuple_Size(space.newlist([]))) api.PyErr_Clear() def test_tuple_resize(self, space, api): - ref_tup = api.PyTuple_New(3) + py_tuple = api.PyTuple_New(3) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - ar[0] = rffi.cast(PyObject, ref_tup) + ar[0] = rffi.cast(PyObject, make_ref(space, py_tuple)) api._PyTuple_Resize(ar, 2) - assert ar[0] == rffi.cast(PyObject, ref_tup) - # ^^^ our _PyTuple_Resize does not actually need to change the ptr so far - assert api.PyTuple_Size(ar[0]) == 2 + py_tuple = from_ref(space, ar[0]) + assert len(py_tuple.wrappeditems) == 2 api._PyTuple_Resize(ar, 10) - assert api.PyTuple_Size(ar[0]) == 10 + py_tuple = from_ref(space, ar[0]) + assert len(py_tuple.wrappeditems) == 10 api.Py_DecRef(ar[0]) lltype.free(ar, flavor='raw') - - def test_tuple_setup(self, space, api): - ref_tup = api.PyTuple_New(2) - ref0 = make_ref(space, space.wrap(123)) - api.PyTuple_SetItem(ref_tup, 0, ref0) - ref1 = make_ref(space, space.wrap(456)) - api.PyTuple_SetItem(ref_tup, 1, ref1) - - w_tup = from_ref(space, ref_tup) - assert space.is_true(space.eq(w_tup, space.wrap((123, 456)))) - api.Py_DecRef(ref_tup) Modified: pypy/branch/fast-forward/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/tupleobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/tupleobject.py Fri Dec 3 19:09:09 2010 @@ -1,144 +1,55 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers, PyObjectFields, - cpython_struct, bootstrap_function) + build_type_checkers) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - borrow_from, make_ref, from_ref, make_typedescr, get_typedescr, Reference, - track_reference) + borrow_from, make_ref, from_ref) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall +from pypy.objspace.std.tupleobject import W_TupleObject -## -## Implementation of PyTupleObject -## =============================== -## -## We have the same problem as PyStringObject: a PyTupleObject can be -## initially used in a read-write way with PyTuple_New(), PyTuple_SetItem() -## and _PyTuple_Resize(). -## -## The 'size' and 'items' fields of a PyTupleObject are always valid. -## Apart from that detail, see the big comment in stringobject.py for -## more information. -## - -ARRAY_OF_PYOBJ = rffi.CArrayPtr(PyObject) -PyTupleObjectStruct = lltype.ForwardReference() -PyTupleObject = lltype.Ptr(PyTupleObjectStruct) -PyTupleObjectFields = PyObjectFields + \ - (("items", ARRAY_OF_PYOBJ), ("size", Py_ssize_t)) -cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) - - at bootstrap_function -def init_tupleobject(space): - "Type description of PyTupleObject" - make_typedescr(space.w_tuple.instancetypedef, - basestruct=PyTupleObject.TO, - attach=tuple_attach, - dealloc=tuple_dealloc, - realize=tuple_realize) PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple") -def new_empty_tuple(space, length): - """ - Allocate a PyTupleObject and its array, but without a corresponding - interpreter object. The array items may be mutated, until - tuple_realize() is called. - """ - typedescr = get_typedescr(space.w_tuple.instancetypedef) - py_obj = typedescr.allocate(space, space.w_tuple) - py_tup = rffi.cast(PyTupleObject, py_obj) - - py_tup.c_items = lltype.malloc(ARRAY_OF_PYOBJ.TO, length, - flavor='raw', zero=True) - py_tup.c_size = length - return py_tup - -def tuple_attach(space, py_obj, w_obj): - """ - Fills a newly allocated PyTupleObject with the given tuple object. - """ - items_w = space.fixedview(w_obj) - py_tup = rffi.cast(PyTupleObject, py_obj) - py_tup.c_items = lltype.nullptr(ARRAY_OF_PYOBJ.TO) - py_tup.c_size = len(items_w) - -def tuple_realize(space, py_obj): - """ - Creates the tuple in the interpreter. The PyTupleObject items array - must not be modified after this call. - """ - py_tup = rffi.cast(PyTupleObject, py_obj) - # If your CPython extension creates a self-referential tuple - # with PyTuple_SetItem(), you loose. - c_items = py_tup.c_items - items_w = [from_ref(space, c_items[i]) for i in range(py_tup.c_size)] - w_obj = space.newtuple(items_w) - track_reference(space, py_obj, w_obj) - return w_obj - - at cpython_api([PyObject], lltype.Void, external=False) -def tuple_dealloc(space, py_obj): - """Frees allocated PyTupleObject resources. - """ - py_tup = rffi.cast(PyTupleObject, py_obj) - if py_tup.c_items: - for i in range(py_tup.c_size): - Py_DecRef(space, py_tup.c_items[i]) - lltype.free(py_tup.c_items, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) - -#_______________________________________________________________________ - @cpython_api([Py_ssize_t], PyObject) def PyTuple_New(space, size): - return rffi.cast(PyObject, new_empty_tuple(space, size)) + return space.newtuple([space.w_None] * size) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PyTuple_SetItem(space, ref, pos, ref_item): - # XXX steals a reference at the level of PyObjects. Don't try to - # XXX call this function with an interpreter object as ref_item! - - # XXX do PyTuple_Check, without forcing ref as an interpreter object - # XXX -- then if it fails it should also steal a reference, test it!!! - ref_tup = rffi.cast(PyTupleObject, ref) - if not ref_tup.c_items: - msg = "PyTuple_SetItem() called on an already-escaped tuple object" - raise OperationError(space.w_SystemError, space.wrap(msg)) - ref_old = ref_tup.c_items[pos] - ref_tup.c_items[pos] = ref_item # SetItem steals a reference! - Py_DecRef(space, ref_old) +def PyTuple_SetItem(space, w_t, pos, w_obj): + if not PyTuple_Check(space, w_t): + # XXX this should also steal a reference, test it!!! + PyErr_BadInternalCall(space) + assert isinstance(w_t, W_TupleObject) + w_t.wrappeditems[pos] = w_obj + Py_DecRef(space, w_obj) # SetItem steals a reference! return 0 @cpython_api([PyObject, Py_ssize_t], PyObject) -def PyTuple_GetItem(space, ref, pos): - # XXX do PyTuple_Check, without forcing ref as an interpreter object - ref_tup = rffi.cast(PyTupleObject, ref) - if ref_tup.c_items: - return Reference(ref_tup.c_items[pos]) # borrowed reference - else: - w_t = from_ref(space, ref) - w_obj = space.getitem(w_t, space.wrap(pos)) - return borrow_from(w_t, w_obj) - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def _PyTuple_Size_Fast(space, ref): - # custom version: it's not a macro, so it can be called from other .py - # files; but it doesn't include PyTuple_Check() - ref_tup = rffi.cast(PyTupleObject, ref) - return ref_tup.c_size +def PyTuple_GetItem(space, w_t, pos): + if not PyTuple_Check(space, w_t): + PyErr_BadInternalCall(space) + assert isinstance(w_t, W_TupleObject) + w_obj = w_t.wrappeditems[pos] + return borrow_from(w_t, w_obj) + + at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) +def PyTuple_GET_SIZE(space, w_t): + """Return the size of the tuple p, which must be non-NULL and point to a tuple; + no error checking is performed. """ + assert isinstance(w_t, W_TupleObject) + return len(w_t.wrappeditems) @cpython_api([PyObject], Py_ssize_t, error=-1) def PyTuple_Size(space, ref): """Take a pointer to a tuple object, and return the size of that tuple.""" - # XXX do PyTuple_Check, without forcing ref as an interpreter object - ref_tup = rffi.cast(PyTupleObject, ref) - return ref_tup.c_size + if not PyTuple_Check(space, ref): + raise OperationError(space.w_TypeError, + space.wrap("expected tuple object")) + return PyTuple_GET_SIZE(space, ref) @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyTuple_Resize(space, refp, newsize): +def _PyTuple_Resize(space, ref, newsize): """Can be used to resize a tuple. newsize will be the new length of the tuple. Because tuples are supposed to be immutable, this should only be used if there is only one reference to the object. Do not use this if the tuple may already @@ -149,22 +60,18 @@ this function. If the object referenced by *p is replaced, the original *p is destroyed. On failure, returns -1 and sets *p to NULL, and raises MemoryError or SystemError.""" - # XXX do PyTuple_Check, without forcing ref as an interpreter object - # XXX -- then if it fails it should reset refp[0] to null - ref_tup = rffi.cast(PyTupleObject, refp[0]) - c_newitems = lltype.malloc(ARRAY_OF_PYOBJ.TO, newsize, - flavor='raw', zero=True) - c_olditems = ref_tup.c_items - if not c_olditems: - msg = "_PyTuple_Resize() called on an already-escaped tuple object" - raise OperationError(space.w_SystemError, space.wrap(msg)) - oldsize = ref_tup.c_size - for i in range(min(oldsize, newsize)): - c_newitems[i] = c_olditems[i] - # decref items deleted by shrinkage - for i in range(newsize, oldsize): - Py_DecRef(space, c_olditems[i]) - ref_tup.c_items = c_newitems - ref_tup.c_size = newsize - lltype.free(c_olditems, flavor='raw') + py_tuple = from_ref(space, ref[0]) + if not PyTuple_Check(space, py_tuple): + PyErr_BadInternalCall(space) + assert isinstance(py_tuple, W_TupleObject) + py_newtuple = PyTuple_New(space, newsize) + + to_cp = newsize + oldsize = len(py_tuple.wrappeditems) + if oldsize < newsize: + to_cp = oldsize + for i in range(to_cp): + py_newtuple.wrappeditems[i] = py_tuple.wrappeditems[i] + Py_DecRef(space, ref[0]) + ref[0] = make_ref(space, py_newtuple) return 0 Modified: pypy/branch/fast-forward/pypy/rlib/rerased.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rerased.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rerased.py Fri Dec 3 19:09:09 2010 @@ -91,7 +91,7 @@ return annmodel.SomeInteger() assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() return annmodel.SomeInstance(clsdef) def specialize_call(self, hop): @@ -108,7 +108,7 @@ def compute_result_annotation(self, s_obj, s_type): assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() s_item = annmodel.SomeInstance(clsdef) return self.bookkeeper.newlist(s_item) Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rbuiltin.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/rbuiltin.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/rbuiltin.py Fri Dec 3 19:09:09 2010 @@ -42,7 +42,7 @@ return hop.genop('cast_pointer', [v_inst], # v_type implicit in r_result resulttype = hop.r_result.lowleveltype) - classdef = s_class.descriptions.keys()[0].getuniqueclassdef() + classdef = s_class.any_description().getuniqueclassdef() return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops) def rtype_builtin_hasattr(hop): Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/rclass.py Fri Dec 3 19:09:09 2010 @@ -392,7 +392,7 @@ source_classdef = source_desc.getclassdef(None) source_repr = getinstancerepr(self.rtyper, source_classdef) assert len(s_func.descriptions) == 1 - funcdesc = s_func.descriptions.keys()[0] + funcdesc, = s_func.descriptions graph = funcdesc.getuniquegraph() FUNCTYPE = FuncType([Ptr(source_repr.object_type)], Void) destrptr = functionptr(FUNCTYPE, graph.name, Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/rdict.py Fri Dec 3 19:09:09 2010 @@ -581,7 +581,7 @@ def ll_dict_lookup_clean(d, hash): # a simplified version of ll_dict_lookup() which assumes that the # key is new, and the dictionary doesn't contain deleted entries. - # It only find the next free slot for the given hash. + # It only finds the next free slot for the given hash. entries = d.entries mask = len(entries) - 1 i = hash & mask Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/rpbc.py Fri Dec 3 19:09:09 2010 @@ -127,7 +127,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(self.rtyper, self.callfamily) assert len(uniquerows) == 1 @@ -166,7 +166,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) def convert_desc(self, funcdesc): @@ -230,7 +230,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness Modified: pypy/branch/fast-forward/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gc/minimark.py Fri Dec 3 19:09:09 2010 @@ -652,8 +652,13 @@ # means recording that they have a smaller size, so that when # moved out of the nursery, they will consume less memory. # In particular, an array with GCFLAG_HAS_CARDS is never resized. + # Also, a nursery object with GCFLAG_HAS_SHADOW is not resized + # either, as this would potentially loose part of the memory in + # the already-allocated shadow. if not self.is_in_nursery(obj): return False + if self.header(obj).tid & GCFLAG_HAS_SHADOW: + return False # size_gc_header = self.gcheaderbuilder.size_gc_header typeid = self.get_type_id(obj) @@ -1423,12 +1428,21 @@ size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + size) - # initialize to an invalid tid *without* GCFLAG_VISITED, - # so that if the object dies before the next minor - # collection, the shadow will stay around but be collected - # by the next major collection. + # Initialize the shadow enough to be considered a + # valid gc object. If the original object stays + # alive at the next minor collection, it will anyway + # be copied over the shadow and overwrite the + # following fields. But if the object dies, then + # the shadow will stay around and only be freed at + # the next major collection, at which point we want + # it to look valid (but ready to be freed). shadow = shadowhdr + size_gc_header - self.header(shadow).tid = 0 + self.header(shadow).tid = self.header(obj).tid + typeid = self.get_type_id(obj) + if self.is_varsize(typeid): + lenofs = self.varsize_offset_to_length(typeid) + (shadow + lenofs).signed[0] = (obj + lenofs).signed[0] + # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.young_objects_shadows.setitem(obj, shadow) # Modified: pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gc/test/test_direct.py Fri Dec 3 19:09:09 2010 @@ -342,6 +342,15 @@ self.gc.collect() assert hash == self.gc.identityhash(self.stackroots[-1]) self.stackroots.pop() + # (7) the same, but the objects are dying young + for i in range(10): + self.gc.collect() + p = self.malloc(VAR, i) + self.stackroots.append(p) + hash1 = self.gc.identityhash(p) + hash2 = self.gc.identityhash(p) + assert hash1 == hash2 + self.stackroots.pop() def test_memory_alignment(self): A1 = lltype.GcArray(lltype.Char) Modified: pypy/branch/fast-forward/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gctransform/boehm.py Fri Dec 3 19:09:09 2010 @@ -120,10 +120,11 @@ fptr = self.annotate_finalizer(d['ll_finalizer'], [llmemory.Address], lltype.Void) elif destrptr: EXC_INSTANCE_TYPE = self.translator.rtyper.exceptiondata.lltype_of_exception_value + typename = TYPE.__name__ def ll_finalizer(addr): exc_instance = llop.gc_fetch_exception(EXC_INSTANCE_TYPE) v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) llop.gc_restore_exception(lltype.Void, exc_instance) fptr = self.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) else: Modified: pypy/branch/fast-forward/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gctransform/framework.py Fri Dec 3 19:09:09 2010 @@ -1204,9 +1204,10 @@ assert not type_contains_pyobjs(TYPE), "not implemented" if destrptr: + typename = TYPE.__name__ def ll_finalizer(addr): v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) fptr = self.transformer.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) Modified: pypy/branch/fast-forward/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gctransform/refcounting.py Fri Dec 3 19:09:09 2010 @@ -227,7 +227,7 @@ # refcount is at zero, temporarily bump it to 1: gcheader.refcount = 1 destr_v = cast_pointer(DESTR_ARG, v) - ll_call_destructor(destrptr, destr_v) + ll_call_destructor(destrptr, destr_v, %r) refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: @@ -239,7 +239,7 @@ pop_alive(exc_instance) # XXX layering of exceptiontransform versus gcpolicy -""" % (body, TYPE._gckind) +""" % (TYPE.__name__, body, TYPE._gckind) else: call_del = None body = '\n'.join(_static_deallocator_body_for_type('v', TYPE)) Modified: pypy/branch/fast-forward/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gctransform/support.py Fri Dec 3 19:09:09 2010 @@ -98,11 +98,15 @@ hop.exception_cannot_occur() return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const) -def ll_call_destructor(destrptr, destr_v): +def ll_call_destructor(destrptr, destr_v, typename): try: destrptr(destr_v) - except: + except Exception, e: try: - os.write(2, "a destructor raised an exception, ignoring it\n") + os.write(2, "a destructor of type ") + os.write(2, typename) + os.write(2, " raised an exception ") + os.write(2, str(e)) + os.write(2, " ignoring it\n") except: pass Modified: pypy/branch/fast-forward/pypy/rpython/ootypesystem/rpbc.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/ootypesystem/rpbc.py (original) +++ pypy/branch/fast-forward/pypy/rpython/ootypesystem/rpbc.py Fri Dec 3 19:09:09 2010 @@ -49,7 +49,7 @@ return hop.genop('runtimenew', [v_class], resulttype=resulttype) def getlowleveltype(self): - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) # if any of the classdefs get the lowleveltype ootype.Class, # we can only pick ootype.Class for us too. Otherwise META. for classdesc in classdescs: @@ -70,7 +70,7 @@ class MethodImplementations(object): def __init__(self, rtyper, methdescs): - samplemdesc = methdescs.iterkeys().next() + samplemdesc = iter(methdescs).next() concretetable, uniquerows = get_concrete_calltable(rtyper, samplemdesc.funcdesc.getcallfamily()) self.row_mapping = {} @@ -117,7 +117,7 @@ concretetable = None # set by _setup_repr_final def _setup_repr_final(self): - sampledesc = self.s_pbc.descriptions.iterkeys().next() + sampledesc = self.s_pbc.any_description() self.concretetable, _ = get_concrete_calltable(self.rtyper, sampledesc.funcdesc.getcallfamily()) Modified: pypy/branch/fast-forward/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/rpbc.py (original) +++ pypy/branch/fast-forward/pypy/rpython/rpbc.py Fri Dec 3 19:09:09 2010 @@ -15,11 +15,10 @@ from pypy.rpython import callparse - def small_cand(rtyper, s_pbc): if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets and \ hasattr(rtyper.type_system.rpbc, 'SmallFunctionSetPBCRepr'): - callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(rtyper, callfamily) if len(uniquerows) == 1 and (not s_pbc.subset_of or small_cand(rtyper, s_pbc.subset_of)): return True @@ -31,7 +30,7 @@ return none_frozen_pbc_repr kind = self.getKind() if issubclass(kind, description.FunctionDesc): - sample = self.descriptions.keys()[0] + sample = self.any_description() callfamily = sample.querycallfamily() if callfamily and callfamily.total_calltable_size > 0: if sample.overridden: @@ -181,7 +180,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None: # a single function self.lowleveltype = Void @@ -207,7 +206,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) ## def function_signatures(self): @@ -322,7 +321,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness @@ -368,7 +367,7 @@ return robject.pyobj_repr def getFrozenPBCRepr(rtyper, s_pbc): - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) assert len(descs) >= 1 if len(descs) == 1 and not s_pbc.can_be_None: return SingleFrozenPBCRepr(descs[0]) @@ -530,7 +529,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper - self.funcdesc = s_pbc.descriptions.keys()[0].funcdesc + self.funcdesc = s_pbc.any_description().funcdesc # a hack to force the underlying function to show up in call_families # (generally not needed, as normalizecalls() should ensure this, @@ -662,7 +661,7 @@ and the ClassRepr of the class which stores this attribute in its vtable. """ - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) access = classdescs[0].queryattrfamily(attrname) for classdesc in classdescs[1:]: access1 = classdesc.queryattrfamily(attrname) @@ -819,7 +818,7 @@ if s_pbc.isNone(): raise TyperError("unsupported: variable of type " "bound-method-object or None") - mdescs = s_pbc.descriptions.keys() + mdescs = list(s_pbc.descriptions) methodname = mdescs[0].name classdef = mdescs[0].selfclassdef flags = mdescs[0].flags Modified: pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/fast-forward/pypy/rpython/tool/rffi_platform.py Fri Dec 3 19:09:09 2010 @@ -673,7 +673,6 @@ C_HEADER = """ #include #include /* for offsetof() */ -#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); Modified: pypy/branch/fast-forward/pypy/tool/error.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/error.py (original) +++ pypy/branch/fast-forward/pypy/tool/error.py Fri Dec 3 19:09:09 2010 @@ -120,7 +120,7 @@ msg.append(" (%s getting at the binding!)" % ( e.__class__.__name__,)) return - for desc in descs.keys(): + for desc in list(descs): func = desc.pyobj if func is None: r = repr(desc) From afa at codespeak.net Fri Dec 3 19:14:04 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 19:14:04 +0100 (CET) Subject: [pypy-svn] r79792 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101203181404.D2471282BF1@codespeak.net> Author: afa Date: Fri Dec 3 19:14:03 2010 New Revision: 79792 Added: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/seq_tests.py - copied, changed from r79790, pypy/branch/fast-forward/lib-python/2.7.0/test/seq_tests.py Log: Port r79750 to the 2.7 test suite Copied: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/seq_tests.py (from r79790, pypy/branch/fast-forward/lib-python/2.7.0/test/seq_tests.py) ============================================================================== --- pypy/branch/fast-forward/lib-python/2.7.0/test/seq_tests.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/seq_tests.py Fri Dec 3 19:14:03 2010 @@ -307,12 +307,18 @@ def test_bigrepeat(self): import sys - if sys.maxint <= 2147483647: - x = self.type2test([0]) - x *= 2**16 - self.assertRaises(MemoryError, x.__mul__, 2**16) - if hasattr(x, '__imul__'): - self.assertRaises(MemoryError, x.__imul__, 2**16) + # we chose an N such as 2**16 * N does not fit into a cpu word + if sys.maxint == 2147483647: + # 32 bit system + N = 2**16 + else: + # 64 bit system + N = 2**48 + x = self.type2test([0]) + x *= 2**16 + self.assertRaises(MemoryError, x.__mul__, N) + if hasattr(x, '__imul__'): + self.assertRaises(MemoryError, x.__imul__, N) def test_subscript(self): a = self.type2test([10, 11]) From fijall at gmail.com Fri Dec 3 20:39:08 2010 From: fijall at gmail.com (Maciej Fijalkowski) Date: Fri, 3 Dec 2010 21:39:08 +0200 Subject: [pypy-svn] r79784 - pypy/branch/arm-backend/pypy/jit/backend/arm/helper In-Reply-To: <20101203162920.99E83282B9E@codespeak.net> References: <20101203162920.99E83282B9E@codespeak.net> Message-ID: Hey david. We usually write tests for such cases and put them into test suite. On Fri, Dec 3, 2010 at 6:29 PM, wrote: > Author: david > Date: Fri Dec ?3 17:29:17 2010 > New Revision: 79784 > > Modified: > ? pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py > Log: > Fix comparison ops based on random test results > > Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py > ============================================================================== > --- pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py ? ?(original) > +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py ? ?Fri Dec ?3 17:29:17 2010 > @@ -45,7 +45,7 @@ > ? ? ? ? regalloc.possibly_free_var(op.result) > > ? ? ? ? if l1.is_imm(): > - ? ? ? ? ? ?ri_op(self.mc, res.value, l0.value, imm=l1.getint(), cond=fcond) > + ? ? ? ? ? ?ri_op(self.mc, res.value, l0.value, imm=l1.value, cond=fcond) > ? ? ? ? else: > ? ? ? ? ? ? rr_op(self.mc, res.value, l0.value, l1.value) > ? ? ? ? return fcond > @@ -74,30 +74,32 @@ > ?def gen_emit_cmp_op(condition, inverse=False): > ? ? def f(self, op, regalloc, fcond): > ? ? ? ? assert fcond is not None > - ? ? ? ?args = op.getarglist() > + ? ? ? ?boxes = list(op.getarglist()) > ? ? ? ? if not inverse: > - ? ? ? ? ? ?arg0 = op.getarg(0) > - ? ? ? ? ? ?arg1 = op.getarg(1) > + ? ? ? ? ? ?arg0, arg1 = boxes > ? ? ? ? else: > - ? ? ? ? ? ?arg0 = op.getarg(1) > - ? ? ? ? ? ?arg1 = op.getarg(0) > + ? ? ? ? ? ?arg1, arg0 = boxes > ? ? ? ? # XXX consider swapping argumentes if arg0 is const > ? ? ? ? imm_a0 = self._check_imm_arg(arg0) > ? ? ? ? imm_a1 = self._check_imm_arg(arg1) > + > + ? ? ? ?l0, box = self._ensure_value_is_boxed(arg0, regalloc, forbidden_vars=boxes) > + ? ? ? ?boxes.append(box) > ? ? ? ? if imm_a1 and not imm_a0: > - ? ? ? ? ? ?l0 = regalloc.make_sure_var_in_reg(arg0, args, imm_fine=False) > - ? ? ? ? ? ?l1 = regalloc.make_sure_var_in_reg(arg1, args) > - ? ? ? ? ? ?res = regalloc.force_allocate_reg(op.result) > - ? ? ? ? ? ?self.mc.CMP_ri(l0.value, imm=l1.getint(), cond=fcond) > + ? ? ? ? ? ?l1 = regalloc.make_sure_var_in_reg(arg1, boxes) > ? ? ? ? else: > - ? ? ? ? ? ?l0 = regalloc.make_sure_var_in_reg(arg0, args, imm_fine=False) > - ? ? ? ? ? ?l1 = regalloc.make_sure_var_in_reg(arg1, args, imm_fine=False) > - ? ? ? ? ? ?res = regalloc.force_allocate_reg(op.result) > - ? ? ? ? ? ?self.mc.CMP_rr(l0.value, l1.value, cond=fcond) > + ? ? ? ? ? ?l1, box = self._ensure_value_is_boxed(arg1, regalloc, forbidden_vars=boxes) > + ? ? ? ? ? ?boxes.append(box) > + ? ? ? ?res = regalloc.force_allocate_reg(op.result) > + ? ? ? ?regalloc.possibly_free_vars(boxes) > + ? ? ? ?regalloc.possibly_free_var(op.result) > > ? ? ? ? inv = c.get_opposite_of(condition) > + ? ? ? ?if l1.is_imm(): > + ? ? ? ? ? ?self.mc.CMP_ri(l0.value, imm=l1.getint(), cond=fcond) > + ? ? ? ?else: > + ? ? ? ? ? ?self.mc.CMP_rr(l0.value, l1.value, cond=fcond) > ? ? ? ? self.mc.MOV_ri(res.value, 1, cond=condition) > ? ? ? ? self.mc.MOV_ri(res.value, 0, cond=inv) > - ? ? ? ?regalloc.possibly_free_vars([arg0, arg1, op.result]) > ? ? ? ? return fcond > ? ? return f > _______________________________________________ > pypy-svn mailing list > pypy-svn at codespeak.net > http://codespeak.net/mailman/listinfo/pypy-svn > From afa at codespeak.net Fri Dec 3 21:36:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 21:36:38 +0100 (CET) Subject: [pypy-svn] r79793 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20101203203638.DB9EE282BDD@codespeak.net> Author: afa Date: Fri Dec 3 21:36:35 2010 New Revision: 79793 Modified: pypy/trunk/pypy/module/cpyext/include/Python.h Log: Add PyFPE_START_PROTECT macros, with the definition they have on almost all CPython builds (except when you configure with ---with-fpectl) Modified: pypy/trunk/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/Python.h (original) +++ pypy/trunk/pypy/module/cpyext/include/Python.h Fri Dec 3 21:36:35 2010 @@ -120,4 +120,8 @@ #define PyDoc_STR(str) "" #endif +/* PyPy does not implement --with-fpectl */ +#define PyFPE_START_PROTECT(err_string, leave_stmt) +#define PyFPE_END_PROTECT(v) + #endif From afa at codespeak.net Fri Dec 3 22:52:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 3 Dec 2010 22:52:53 +0100 (CET) Subject: [pypy-svn] r79794 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20101203215253.62310282BE0@codespeak.net> Author: afa Date: Fri Dec 3 22:52:49 2010 New Revision: 79794 Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Log: Implement PyErr_NormalizeException() Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Fri Dec 3 22:52:49 2010 @@ -5,7 +5,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, Py_DecRef, borrow_from) + PyObject, PyObjectP, make_ref, from_ref, Py_DecRef, borrow_from) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from pypy.rlib.rposix import get_errno @@ -80,6 +80,21 @@ Py_DecRef(space, w_value) Py_DecRef(space, w_traceback) + at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) +def PyErr_NormalizeException(space, exc_p, val_p, tb_p): + """Under certain circumstances, the values returned by PyErr_Fetch() below + can be "unnormalized", meaning that *exc is a class object but *val is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance.""" + operr = OperationError(from_ref(space, exc_p[0]), + from_ref(space, val_p[0])) + operr.normalize_exception(space) + Py_DecRef(space, exc_p[0]) + Py_DecRef(space, val_p[0]) + exc_p[0] = make_ref(space, operr.w_type) + val_p[0] = make_ref(space, operr.get_w_value(space)) + @cpython_api([], lltype.Void) def PyErr_BadArgument(space): """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Fri Dec 3 22:52:49 2010 @@ -668,15 +668,6 @@ """ raise NotImplementedError - at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) -def PyErr_NormalizeException(space, exc, val, tb): - """Under certain circumstances, the values returned by PyErr_Fetch() below - can be "unnormalized", meaning that *exc is a class object but *val is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance.""" - raise NotImplementedError - @cpython_api([PyObject, rffi.CCHARP], PyObject) def PyErr_SetFromErrnoWithFilename(space, type, filename): """Similar to PyErr_SetFromErrno(), with the additional behavior that if Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Fri Dec 3 22:52:49 2010 @@ -129,6 +129,42 @@ ]) assert module.check_error() + + def test_normalize(self): + module = self.import_extension('foo', [ + ("check_error", "METH_NOARGS", + ''' + PyObject *type, *val, *tb; + PyErr_SetString(PyExc_TypeError, "message"); + + PyErr_Fetch(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (!PyString_Check(val)) + Py_RETURN_FALSE; + /* Normalize */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + /* Normalize again */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + PyErr_Restore(type, val, tb); + if (!PyErr_Occurred()) + Py_RETURN_FALSE; + Py_RETURN_TRUE; + ''' + ), + ]) + assert module.check_error() + def test_SetFromErrno(self): import sys if sys.platform != 'win32': From david at codespeak.net Sat Dec 4 09:25:22 2010 From: david at codespeak.net (david at codespeak.net) Date: Sat, 4 Dec 2010 09:25:22 +0100 (CET) Subject: [pypy-svn] r79795 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . test Message-ID: <20101204082522.7F2BA282B9C@codespeak.net> Author: david Date: Sat Dec 4 09:25:20 2010 New Revision: 79795 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Log: Fixes for getfield, setfield and for rshift operations with constant arguments Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Sat Dec 4 09:25:20 2010 @@ -5,8 +5,9 @@ from pypy.jit.backend.arm.codebuilder import ARMv7Builder, ARMv7InMemoryBuilder from pypy.jit.backend.arm.regalloc import ARMRegisterManager, ARMFrameManager from pypy.jit.backend.llsupport.regalloc import compute_vars_longevity, TempBox -from pypy.jit.metainterp.history import (Const, ConstInt, BoxInt, AbstractFailDescr, - INT, REF, FLOAT) +from pypy.jit.metainterp.history import (Const, ConstInt, ConstPtr, + BoxInt, BoxPtr, AbstractFailDescr, + INT, REF, FLOAT) from pypy.jit.metainterp.resoperation import rop from pypy.rlib import rgc from pypy.rpython.annlowlevel import llhelper @@ -454,6 +455,8 @@ if isinstance(thing, Const): if isinstance(thing, ConstInt): box = BoxInt() + elif isinstance(thing, ConstPtr): + box = BoxPtr() else: box = TempBox() loc = regalloc.force_allocate_reg(box, Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Sat Dec 4 09:25:20 2010 @@ -146,8 +146,8 @@ emit_op_int_or = gen_emit_op_ri('ORR') emit_op_int_xor = gen_emit_op_ri('EOR') emit_op_int_lshift = gen_emit_op_ri('LSL', imm_size=0x1F, allow_zero=False, commutative=False) - emit_op_int_rshift = gen_emit_op_ri('ASR', imm_size=0x1F, commutative=False) - emit_op_uint_rshift = gen_emit_op_ri('LSR', imm_size=0x1F, commutative=False) + emit_op_int_rshift = gen_emit_op_ri('ASR', imm_size=0x1F, allow_zero=False, commutative=False) + emit_op_uint_rshift = gen_emit_op_ri('LSR', imm_size=0x1F, allow_zero=False, commutative=False) emit_op_int_lt = gen_emit_cmp_op(c.LT) emit_op_int_le = gen_emit_cmp_op(c.LE) @@ -444,13 +444,16 @@ _mixin_ = True def emit_op_setfield_gc(self, op, regalloc, fcond): - a0 = op.getarg(0) - a1 = op.getarg(1) + boxes = list(op.getarglist()) + a0, a1 = boxes ofs, size, ptr = self._unpack_fielddescr(op.getdescr()) #ofs_loc = regalloc.make_sure_var_in_reg(ConstInt(ofs)) #size_loc = regalloc.make_sure_var_in_reg(ofs) - base_loc = regalloc.make_sure_var_in_reg(a0, imm_fine=False) - value_loc = regalloc.make_sure_var_in_reg(a1, [a0], imm_fine=False) + base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc, boxes) + boxes.append(base_box) + value_loc, value_box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(value_box) + regalloc.possibly_free_vars(boxes) if size == 4: self.mc.STR_ri(value_loc.value, base_loc.value, ofs) elif size == 2: @@ -467,8 +470,12 @@ a0 = op.getarg(0) ofs, size, ptr = self._unpack_fielddescr(op.getdescr()) # ofs_loc = regalloc.make_sure_var_in_reg(ConstInt(ofs)) - base_loc = regalloc.make_sure_var_in_reg(a0, imm_fine=False) + base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc) res = regalloc.force_allocate_reg(op.result, [a0]) + regalloc.possibly_free_var(a0) + regalloc.possibly_free_var(base_box) + regalloc.possibly_free_var(op.result) + if size == 4: self.mc.LDR_ri(res.value, base_loc.value, ofs) elif size == 2: Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_generated.py Sat Dec 4 09:25:20 2010 @@ -555,3 +555,59 @@ assert cpu.get_latest_value_int(3) == 357913940 assert cpu.get_latest_value_int(4) == 16 assert cpu.get_latest_value_int(5) == -5 + + def test_wrong_result2(self): + # block length 10 + # random seed 1 + f1 = BasicFailDescr(1) + f2 = BasicFailDescr(2) + f3 = BasicFailDescr(3) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + cpu = CPU(None, None) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + operations = [ + ResOperation(rop.INT_LE, [v6, v1], v11), + ResOperation(rop.SAME_AS, [ConstInt(-14)], v12), + ResOperation(rop.INT_ADD, [ConstInt(24), v4], v13), + ResOperation(rop.UINT_RSHIFT, [v6, ConstInt(0)], v14), + ResOperation(rop.GUARD_VALUE, [v14, ConstInt(1)], None, descr=f3), + ResOperation(rop.INT_MUL, [v13, ConstInt(12)], v15), + ResOperation(rop.GUARD_FALSE, [v11], None, descr=f1), + ResOperation(rop.FINISH, [v2, v3, v5, v7, v10, v8, v9], None, descr=f2), + ] + operations[-2].setfailargs([v4, v10, v3, v9, v14, v2]) + operations[4].setfailargs([v14]) + looptoken = LoopToken() + cpu.compile_loop(inputargs, operations, looptoken) + cpu.set_future_value_int(0, 14) + cpu.set_future_value_int(1, -20) + cpu.set_future_value_int(2, 18) + cpu.set_future_value_int(3, -2058005163) + cpu.set_future_value_int(4, 6) + cpu.set_future_value_int(5, 1) + cpu.set_future_value_int(6, -16) + cpu.set_future_value_int(7, 11) + cpu.set_future_value_int(8, 0) + cpu.set_future_value_int(9, 19) + op = cpu.execute_token(looptoken) + assert op.identifier == 1 + assert cpu.get_latest_value_int(0) == -2058005163 + assert cpu.get_latest_value_int(1) == 19 + assert cpu.get_latest_value_int(2) == 18 + assert cpu.get_latest_value_int(3) == 0 + assert cpu.get_latest_value_int(4) == 1 + assert cpu.get_latest_value_int(5) == -20 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Sat Dec 4 09:25:20 2010 @@ -9,12 +9,12 @@ # one OPERATIONS = test_random.OPERATIONS[:] -#for i in range(4): # make more common -# OPERATIONS.append(test_ll_random.GetFieldOperation(rop.GETFIELD_GC)) -# OPERATIONS.append(test_ll_random.GetFieldOperation(rop.GETFIELD_GC)) -# OPERATIONS.append(test_ll_random.SetFieldOperation(rop.SETFIELD_GC)) -# #OPERATIONS.append(test_ll_random.NewOperation(rop.NEW)) -# #OPERATIONS.append(test_ll_random.NewOperation(rop.NEW_WITH_VTABLE)) +for i in range(4): # make more common + OPERATIONS.append(test_ll_random.GetFieldOperation(rop.GETFIELD_GC)) + OPERATIONS.append(test_ll_random.GetFieldOperation(rop.GETFIELD_GC)) + OPERATIONS.append(test_ll_random.SetFieldOperation(rop.SETFIELD_GC)) + OPERATIONS.append(test_ll_random.NewOperation(rop.NEW)) + OPERATIONS.append(test_ll_random.NewOperation(rop.NEW_WITH_VTABLE)) # # OPERATIONS.append(test_ll_random.GetArrayItemOperation(rop.GETARRAYITEM_GC)) # OPERATIONS.append(test_ll_random.GetArrayItemOperation(rop.GETARRAYITEM_GC)) From david at codespeak.net Sat Dec 4 10:43:09 2010 From: david at codespeak.net (david at codespeak.net) Date: Sat, 4 Dec 2010 10:43:09 +0100 (CET) Subject: [pypy-svn] r79797 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . test Message-ID: <20101204094309.45E4F282B9C@codespeak.net> Author: david Date: Sat Dec 4 10:43:06 2010 New Revision: 79797 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Log: Correctly support const args in array set/get and len operations and refactor a bit Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Sat Dec 4 10:43:06 2010 @@ -514,37 +514,40 @@ assert isinstance(arraydescr, BaseArrayDescr) ofs = arraydescr.get_ofs_length(self.cpu.translate_support_code) arg = op.getarg(0) - base_loc = regalloc.make_sure_var_in_reg(arg, imm_fine=False) - regalloc.possibly_free_vars_for_op(op) - res = regalloc.force_allocate_reg(op.result, forbidden_vars=[arg]) + base_loc, base_box = self._ensure_value_is_boxed(arg, regalloc) + res = regalloc.force_allocate_reg(op.result, forbidden_vars=[arg, base_box]) + regalloc.possibly_free_vars([arg, base_box, op.result]) self.mc.LDR_ri(res.value, base_loc.value, ofs) - regalloc.possibly_free_var(arg) - regalloc.possibly_free_var(op.result) return fcond def emit_op_setarrayitem_gc(self, op, regalloc, fcond): - a0 = op.getarg(0) - a1 = op.getarg(1) - a2 = op.getarg(2) + a0, a1, a2 = boxes = list(op.getarglist()) _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr()) - base_loc = regalloc.make_sure_var_in_reg(a0, imm_fine=False) - ofs_loc = regalloc.make_sure_var_in_reg(a1, imm_fine=False) + base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc, boxes) + boxes.append(base_box) + ofs_loc, ofs_box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(ofs_box) #XXX check if imm would be fine here - value_loc = regalloc.make_sure_var_in_reg(a2, imm_fine=False) + value_loc, value_box = self._ensure_value_is_boxed(a2, regalloc, boxes) + boxes.append(value_box) + regalloc.possibly_free_vars(boxes) if scale > 0: - self.mc.LSL_ri(ofs_loc.value, ofs_loc.value, scale) + self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale) + else: + self.mc.MOV_rr(r.ip.value, ofs_loc.value) + if ofs > 0: - self.mc.ADD_ri(ofs_loc.value, ofs_loc.value, ofs) + self.mc.ADD_ri(r.ip.value, r.ip.value, ofs) if scale == 2: - self.mc.STR_rr(value_loc.value, base_loc.value, ofs_loc.value, cond=fcond) + self.mc.STR_rr(value_loc.value, base_loc.value, r.ip.value, cond=fcond) elif scale == 1: - self.mc.STRH_rr(value_loc.value, base_loc.value, ofs_loc.value, cond=fcond) + self.mc.STRH_rr(value_loc.value, base_loc.value, r.ip.value, cond=fcond) elif scale == 0: - self.mc.STRB_rr(value_loc.value, base_loc.value, ofs_loc.value, cond=fcond) + self.mc.STRB_rr(value_loc.value, base_loc.value, r.ip.value, cond=fcond) else: assert 0 return fcond @@ -552,25 +555,30 @@ emit_op_setarrayitem_raw = emit_op_setarrayitem_gc def emit_op_getarrayitem_gc(self, op, regalloc, fcond): - a0 = op.getarg(0) - a1 = op.getarg(1) + a0, a1 = boxes = list(op.getarglist()) _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr()) - base_loc = regalloc.make_sure_var_in_reg(a0, imm_fine=False) - ofs_loc = regalloc.make_sure_var_in_reg(a1, imm_fine=False) + base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc, boxes) + boxes.append(base_box) + ofs_loc, ofs_box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(ofs_box) res = regalloc.force_allocate_reg(op.result) + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) if scale > 0: - self.mc.LSL_ri(ofs_loc.value, ofs_loc.value, scale) + self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale) + else: + self.mc.MOV_rr(r.ip.value, ofs_loc.value) if ofs > 0: - self.mc.ADD_ri(ofs_loc.value, ofs_loc.value, imm=ofs) + self.mc.ADD_ri(r.ip.value, r.ip.value, imm=ofs) if scale == 2: - self.mc.LDR_rr(res.value, base_loc.value, ofs_loc.value, cond=fcond) + self.mc.LDR_rr(res.value, base_loc.value, r.ip.value, cond=fcond) elif scale == 1: - self.mc.LDRH_rr(res.value, base_loc.value, ofs_loc.value, cond=fcond) + self.mc.LDRH_rr(res.value, base_loc.value, r.ip.value, cond=fcond) elif scale == 0: - self.mc.LDRB_rr(res.value, base_loc.value, ofs_loc.value, cond=fcond) + self.mc.LDRB_rr(res.value, base_loc.value, r.ip.value, cond=fcond) else: assert 0 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Sat Dec 4 10:43:06 2010 @@ -15,12 +15,12 @@ OPERATIONS.append(test_ll_random.SetFieldOperation(rop.SETFIELD_GC)) OPERATIONS.append(test_ll_random.NewOperation(rop.NEW)) OPERATIONS.append(test_ll_random.NewOperation(rop.NEW_WITH_VTABLE)) -# -# OPERATIONS.append(test_ll_random.GetArrayItemOperation(rop.GETARRAYITEM_GC)) -# OPERATIONS.append(test_ll_random.GetArrayItemOperation(rop.GETARRAYITEM_GC)) -# OPERATIONS.append(test_ll_random.SetArrayItemOperation(rop.SETARRAYITEM_GC)) -# #OPERATIONS.append(test_ll_random.NewArrayOperation(rop.NEW_ARRAY)) -# OPERATIONS.append(test_ll_random.ArrayLenOperation(rop.ARRAYLEN_GC)) + + OPERATIONS.append(test_ll_random.GetArrayItemOperation(rop.GETARRAYITEM_GC)) + OPERATIONS.append(test_ll_random.GetArrayItemOperation(rop.GETARRAYITEM_GC)) + OPERATIONS.append(test_ll_random.SetArrayItemOperation(rop.SETARRAYITEM_GC)) + #OPERATIONS.append(test_ll_random.NewArrayOperation(rop.NEW_ARRAY)) + OPERATIONS.append(test_ll_random.ArrayLenOperation(rop.ARRAYLEN_GC)) # #OPERATIONS.append(test_ll_random.NewStrOperation(rop.NEWSTR)) # #OPERATIONS.append(test_ll_random.NewUnicodeOperation(rop.NEWUNICODE)) # OPERATIONS.append(test_ll_random.StrGetItemOperation(rop.STRGETITEM)) From hakanardo at codespeak.net Sat Dec 4 10:55:14 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 4 Dec 2010 10:55:14 +0100 (CET) Subject: [pypy-svn] r79798 - in pypy/branch/jit-unroll-loops/pypy/jit/metainterp: optimizeopt test Message-ID: <20101204095514.6F14D282B9C@codespeak.net> Author: hakanardo Date: Sat Dec 4 10:55:12 2010 New Revision: 79798 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_virtual.py Log: Workaraound forceing failargs Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py Sat Dec 4 10:55:12 2010 @@ -399,6 +399,13 @@ newop = inliner.inline_op(op) if not dryrun: + # FIXME: Emit a proper guard instead to move these + # forceings into the the small bridge back to the preamble + if newop.is_guard(): + for box in newop.getfailargs(): + if box in self.optimizer.values: + box = self.optimizer.values[box].force_box() + self.emit_operation(newop) else: if not self.is_emittable(newop): Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py Sat Dec 4 10:55:12 2010 @@ -1840,6 +1840,31 @@ 'int_add': 1, 'int_mul': 1, 'int_sub': 2, 'int_gt': 2, 'jump': 2}) + def test_specialied_bridge(self): + myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'res']) + class A(Base): + def __init__(self, val): + self.val = val + def binop(self, other): + return A(self.val + other.val) + def f(x, y): + res = A(0) + while y > 0: + myjitdriver.can_enter_jit(y=y, x=x, res=res) + myjitdriver.jit_merge_point(y=y, x=x, res=res) + res = res.binop(A(y)) + if y<7: + res = x + y -= 1 + return res + def g(x, y): + a1 = f(A(x), y) + a2 = f(A(x), y) + assert a1.val == a2.val + return a1.val + res = self.meta_interp(g, [6, 14]) + assert res == g(6, 14) + def test_multiple_specialied_zigzag(self): myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'res']) class Base: Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_virtual.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_virtual.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_virtual.py Sat Dec 4 10:55:12 2010 @@ -239,6 +239,32 @@ assert res == 78 self.check_loops(new_with_vtable=0, new=0) + def test_specialied_bridge(self): + myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'res']) + class A: + def __init__(self, val): + self.val = val + def binop(self, other): + return A(self.val + other.val) + def f(x, y): + res = A(0) + while y > 0: + myjitdriver.can_enter_jit(y=y, x=x, res=res) + myjitdriver.jit_merge_point(y=y, x=x, res=res) + res = res.binop(A(y)) + if y<7: + res = x + x = A(1) + y -= 1 + return res + def g(x, y): + a1 = f(A(x), y) + a2 = f(A(x), y) + assert a1.val == a2.val + return a1.val + res = self.meta_interp(g, [6, 14]) + assert res == g(6, 14) + def test_both_virtual_and_field_variable(self): myjitdriver = JitDriver(greens = [], reds = ['n']) class Foo(object): From hakanardo at codespeak.net Sat Dec 4 11:10:41 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 4 Dec 2010 11:10:41 +0100 (CET) Subject: [pypy-svn] r79799 - in pypy/branch/jit-unroll-loops/pypy/jit: metainterp/test tl Message-ID: <20101204101041.D60EB282B9C@codespeak.net> Author: hakanardo Date: Sat Dec 4 11:10:40 2010 New Revision: 79799 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py pypy/branch/jit-unroll-loops/pypy/jit/tl/pypyjit_demo.py Log: typo Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py Sat Dec 4 11:10:40 2010 @@ -1842,7 +1842,7 @@ def test_specialied_bridge(self): myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'res']) - class A(Base): + class A: def __init__(self, val): self.val = val def binop(self, other): Modified: pypy/branch/jit-unroll-loops/pypy/jit/tl/pypyjit_demo.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/tl/pypyjit_demo.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/tl/pypyjit_demo.py Sat Dec 4 11:10:40 2010 @@ -1,107 +1,18 @@ -## base = object - -## class Number(base): -## __slots__ = ('val', ) -## def __init__(self, val=0): -## self.val = val - -## def __add__(self, other): -## if not isinstance(other, int): -## other = other.val -## return Number(val=self.val + other) - -## def __cmp__(self, other): -## val = self.val -## if not isinstance(other, int): -## other = other.val -## return cmp(val, other) - -## def __nonzero__(self): -## return bool(self.val) - -## def g(x, inc=2): -## return x + inc - -## def f(n, x, inc): -## while x < n: -## x = g(x, inc=1) -## return x - -## import time -## #t1 = time.time() -## #f(10000000, Number(), 1) -## #t2 = time.time() -## #print t2 - t1 -## t1 = time.time() -## f(10000000, 0, 1) -## t2 = time.time() -## print t2 - t1 try: -## from array import array - -## def coords(w,h): -## y = 0 -## while y < h: -## x = 0 -## while x < w: -## yield x,y -## x += 1 -## y += 1 - -## def f(img): -## sa=0 -## for x, y in coords(4,4): -## sa += x * y -## return sa - -## #img=array('h',(1,2,3,4)) -## print f(3) - -## from array import array -## class Circular(array): -## def __new__(cls): -## self = array.__new__(cls, 'i', range(16)) -## return self -## def __getitem__(self, i): -## #assert self.__len__() == 16 -## return array.__getitem__(self, i & 15) - -## def main(): -## buf = Circular() -## i = 10 -## sa = 0 -## while i < 20: -## #sa += buf[i-2] + buf[i-1] + buf[i] + buf[i+1] + buf[i+2] -## sa += buf[i] -## i += 1 -## return sa - import pypyjit pypyjit.set_param(threshold=3, inlining=True) -## print main() def main(): - i=2 - sa=0 - while i < 10: - #sa+=max(range(i)) - a = range - b = max([i]) + i=a=0 + while i<10: i+=1 - return sa + a+=1 + return a + print main() except Exception, e: print "Exception: ", type(e) print e -## def f(): -## a=7 -## i=0 -## while i<4: -## if i<0: break -## if i<0: break -## i+=1 - -## f() From david at codespeak.net Sat Dec 4 13:32:48 2010 From: david at codespeak.net (david at codespeak.net) Date: Sat, 4 Dec 2010 13:32:48 +0100 (CET) Subject: [pypy-svn] r79800 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . test Message-ID: <20101204123248.76754282B9C@codespeak.net> Author: david Date: Sat Dec 4 13:32:45 2010 New Revision: 79800 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Log: Refactor string and unicode operations and make sure const arguments work as expected Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Sat Dec 4 13:32:45 2010 @@ -611,14 +611,24 @@ _mixin_ = True def emit_op_strlen(self, op, regalloc, fcond): - l0 = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) - regalloc.possibly_free_vars_for_op(op) - res = regalloc.force_allocate_reg(op.result) - if op.result: - regalloc.possibly_free_var(op.result) + l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) + boxes = [box] + + res = regalloc.force_allocate_reg(op.result, boxes) + boxes.append(op.result) + basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) - l1 = regalloc.make_sure_var_in_reg(ConstInt(ofs_length)) + imm_ofs = self._check_imm_arg(ofs_length) + + if imm_ofs: + l1 = regalloc.make_sure_var_in_reg(ConstInt(ofs_length), boxes) + else: + l1, box1 = self._ensure_value_is_boxed(ConstInt(ofs_length), regalloc, boxes) + boxes.append(box1) + + regalloc.possibly_free_vars(boxes) + if l1.is_imm(): self.mc.LDR_ri(res.value, l0.value, l1.getint(), cond=fcond) else: @@ -626,46 +636,56 @@ return fcond def emit_op_strgetitem(self, op, regalloc, fcond): - base_loc = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) - ofs_loc = regalloc.make_sure_var_in_reg(op.getarg(1)) - t = TempBox() - temp = regalloc.force_allocate_reg(t) + boxes = list(op.getarglist()) + base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc) + boxes.append(box) + + a1 = boxes[1] + imm_a1 = self._check_imm_arg(a1) + if imm_a1: + ofs_loc = regalloc.make_sure_var_in_reg(a1, boxes) + else: + ofs_loc, box = self._ensure_value_is_boxed(a1, regalloc, boxes) + boxes.append(box) res = regalloc.force_allocate_reg(op.result) - regalloc.possibly_free_vars_for_op(op) - regalloc.possibly_free_var(t) + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 if ofs_loc.is_imm(): - self.mc.ADD_ri(temp.value, base_loc.value, ofs_loc.getint(), cond=fcond) + self.mc.ADD_ri(r.ip.value, base_loc.value, ofs_loc.getint(), cond=fcond) else: - self.mc.ADD_rr(temp.value, base_loc.value, ofs_loc.value, cond=fcond) + self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value, cond=fcond) - self.mc.LDRB_ri(res.value, temp.value, basesize, cond=fcond) + self.mc.LDRB_ri(res.value, r.ip.value, basesize, cond=fcond) return fcond def emit_op_strsetitem(self, op, regalloc, fcond): - base_loc = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) - ofs_loc = regalloc.make_sure_var_in_reg(op.getarg(1)) - value_loc = regalloc.make_sure_var_in_reg(op.getarg(2)) - t = TempBox() - temp = regalloc.force_allocate_reg(t) - regalloc.possibly_free_vars_for_op(op) - regalloc.possibly_free_var(t) - if op.result: - regalloc.possibly_free_var(op.result) + boxes = list(op.getarglist()) + + base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) + boxes.append(box) + + ofs_loc, box = self._ensure_value_is_boxed(boxes[1], regalloc, boxes) + boxes.append(box) + + value_loc, box = self._ensure_value_is_boxed(boxes[2], regalloc, boxes) + boxes.append(box) + + regalloc.possibly_free_vars(boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 if ofs_loc.is_imm(): - self.mc.ADD_ri(temp.value, base_loc.value, ofs_loc.getint(), cond=fcond) + self.mc.ADD_ri(r.ip.value, base_loc.value, ofs_loc.getint(), cond=fcond) else: - self.mc.ADD_rr(temp.value, base_loc.value, ofs_loc.value, cond=fcond) + self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value, cond=fcond) - self.mc.STRB_ri(value_loc.value, temp.value, basesize, cond=fcond) + self.mc.STRB_ri(value_loc.value, r.ip.value, basesize, cond=fcond) return fcond #from ../x86/regalloc.py:928 ff. @@ -776,13 +796,22 @@ _mixin_ = True def emit_op_unicodelen(self, op, regalloc, fcond): - l0 = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) - regalloc.possibly_free_vars_for_op(op) - res = regalloc.force_allocate_reg(op.result) - regalloc.possibly_free_var(op.result) + l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) + boxes = [box] + res = regalloc.force_allocate_reg(op.result, boxes) + boxes.append(op.result) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) - l1 = regalloc.make_sure_var_in_reg(ConstInt(ofs_length)) + imm_ofs = self._check_imm_arg(ofs_length) + + if imm_ofs: + l1 = regalloc.make_sure_var_in_reg(ConstInt(ofs_length), boxes) + else: + l1, box1 = self._ensure_value_is_boxed(ConstInt(ofs_length), regalloc, boxes) + boxes.append(box1) + regalloc.possibly_free_vars(boxes) + + # XXX merge with strlen if l1.is_imm(): self.mc.LDR_ri(res.value, l0.value, l1.getint(), cond=fcond) else: @@ -790,51 +819,57 @@ return fcond def emit_op_unicodegetitem(self, op, regalloc, fcond): - base_loc = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) - ofs_loc = regalloc.make_sure_var_in_reg(op.getarg(1), imm_fine=False) - t = TempBox() - temp = regalloc.force_allocate_reg(t) + boxes = list(op.getarglist()) + + base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) + boxes = [box] + + ofs_loc, box = self._ensure_value_is_boxed(boxes[1], regalloc, boxes) + boxes.append(box) + res = regalloc.force_allocate_reg(op.result) - regalloc.possibly_free_vars_for_op(op) - regalloc.possibly_free_var(t) - if op.result: - regalloc.possibly_free_var(op.result) + + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize/2 - self.mc.ADD_rr(temp.value, base_loc.value, ofs_loc.value, cond=fcond, + self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value, cond=fcond, imm=scale, shifttype=shift.LSL) if scale == 2: - self.mc.LDR_ri(res.value, temp.value, basesize, cond=fcond) + self.mc.LDR_ri(res.value, r.ip.value, basesize, cond=fcond) elif scale == 1: - self.mc.LDRH_ri(res.value, temp.value, basesize, cond=fcond) + self.mc.LDRH_ri(res.value, r.ip.value, basesize, cond=fcond) else: assert 0, itemsize return fcond def emit_op_unicodesetitem(self, op, regalloc, fcond): - base_loc = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) - ofs_loc = regalloc.make_sure_var_in_reg(op.getarg(1)) - value_loc = regalloc.make_sure_var_in_reg(op.getarg(2)) - t = TempBox() - temp = regalloc.force_allocate_reg(t) - regalloc.possibly_free_vars_for_op(op) - regalloc.possibly_free_var(t) - if op.result: - regalloc.possibly_free_var(op.result) + boxes = list(op.getarglist()) + + base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) + boxes.append(box) + + ofs_loc, box = self._ensure_value_is_boxed(boxes[1], regalloc, boxes) + boxes.append(box) + + value_loc, box = self._ensure_value_is_boxed(boxes[2], regalloc, boxes) + boxes.append(box) + + regalloc.possibly_free_vars(boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize/2 - self.mc.ADD_rr(temp.value, base_loc.value, ofs_loc.value, cond=fcond, + self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value, cond=fcond, imm=scale, shifttype=shift.LSL) if scale == 2: - self.mc.STR_ri(value_loc.value, temp.value, basesize, cond=fcond) + self.mc.STR_ri(value_loc.value, r.ip.value, basesize, cond=fcond) elif scale == 1: - self.mc.STRH_ri(value_loc.value, temp.value, basesize, cond=fcond) + self.mc.STRH_ri(value_loc.value, r.ip.value, basesize, cond=fcond) else: assert 0, itemsize Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Sat Dec 4 13:32:45 2010 @@ -23,21 +23,21 @@ OPERATIONS.append(test_ll_random.ArrayLenOperation(rop.ARRAYLEN_GC)) # #OPERATIONS.append(test_ll_random.NewStrOperation(rop.NEWSTR)) # #OPERATIONS.append(test_ll_random.NewUnicodeOperation(rop.NEWUNICODE)) -# OPERATIONS.append(test_ll_random.StrGetItemOperation(rop.STRGETITEM)) -# OPERATIONS.append(test_ll_random.UnicodeGetItemOperation(rop.UNICODEGETITEM)) -# OPERATIONS.append(test_ll_random.StrSetItemOperation(rop.STRSETITEM)) -# OPERATIONS.append(test_ll_random.UnicodeSetItemOperation(rop.UNICODESETITEM)) -# OPERATIONS.append(test_ll_random.StrLenOperation(rop.STRLEN)) -# OPERATIONS.append(test_ll_random.UnicodeLenOperation(rop.UNICODELEN)) + OPERATIONS.append(test_ll_random.StrGetItemOperation(rop.STRGETITEM)) + OPERATIONS.append(test_ll_random.UnicodeGetItemOperation(rop.UNICODEGETITEM)) + OPERATIONS.append(test_ll_random.StrSetItemOperation(rop.STRSETITEM)) + OPERATIONS.append(test_ll_random.UnicodeSetItemOperation(rop.UNICODESETITEM)) + OPERATIONS.append(test_ll_random.StrLenOperation(rop.STRLEN)) + OPERATIONS.append(test_ll_random.UnicodeLenOperation(rop.UNICODELEN)) # -#for i in range(2): - #OPERATIONS.append(test_ll_random.GuardClassOperation(rop.GUARD_CLASS)) +for i in range(2): + OPERATIONS.append(test_ll_random.GuardClassOperation(rop.GUARD_CLASS)) #OPERATIONS.append(test_ll_random.CallOperation(rop.CALL)) #OPERATIONS.append(test_ll_random.RaisingCallOperation(rop.CALL)) #OPERATIONS.append(test_ll_random.RaisingCallOperationGuardNoException(rop.CALL)) #OPERATIONS.append(test_ll_random.RaisingCallOperationWrongGuardException(rop.CALL)) #OPERATIONS.append(test_ll_random.CallOperationException(rop.CALL)) -#OPERATIONS.append(test_ll_random.GuardNonNullClassOperation(rop.GUARD_NONNULL_CLASS)) +OPERATIONS.append(test_ll_random.GuardNonNullClassOperation(rop.GUARD_NONNULL_CLASS)) From hakanardo at codespeak.net Sat Dec 4 14:19:04 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 4 Dec 2010 14:19:04 +0100 (CET) Subject: [pypy-svn] r79801 - in pypy/branch/jit-unroll-loops/pypy/jit/metainterp: optimizeopt test Message-ID: <20101204131904.D4B81282BE0@codespeak.net> Author: hakanardo Date: Sat Dec 4 14:19:02 2010 New Revision: 79801 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py Log: No Consts allowed in the failargs Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/unroll.py Sat Dec 4 14:19:02 2010 @@ -402,9 +402,22 @@ # FIXME: Emit a proper guard instead to move these # forceings into the the small bridge back to the preamble if newop.is_guard(): - for box in newop.getfailargs(): + failargs = newop.getfailargs() + for i in range(len(failargs)): + box = failargs[i] if box in self.optimizer.values: - box = self.optimizer.values[box].force_box() + value = self.optimizer.values[box] + if value.is_constant(): + newbox = box.clonebox() + op = ResOperation(rop.SAME_AS, + [value.force_box()], newbox) + self.optimizer.newoperations.append(op) + box = newbox + else: + box = value.force_box() + failargs[i] = box + newop.setfailargs(failargs) + self.emit_operation(newop) else: Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py Sat Dec 4 14:19:02 2010 @@ -1865,6 +1865,33 @@ res = self.meta_interp(g, [6, 14]) assert res == g(6, 14) + def test_specialied_bridge_const(self): + myjitdriver = JitDriver(greens = [], reds = ['y', 'const', 'x', 'res']) + class A: + def __init__(self, val): + self.val = val + def binop(self, other): + return A(self.val + other.val) + def f(x, y): + res = A(0) + const = 7 + while y > 0: + myjitdriver.can_enter_jit(y=y, x=x, res=res, const=const) + myjitdriver.jit_merge_point(y=y, x=x, res=res, const=const) + const = hint(const, promote=True) + res = res.binop(A(const)) + if y<7: + res = x + y -= 1 + return res + def g(x, y): + a1 = f(A(x), y) + a2 = f(A(x), y) + assert a1.val == a2.val + return a1.val + res = self.meta_interp(g, [6, 14]) + assert res == g(6, 14) + def test_multiple_specialied_zigzag(self): myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'res']) class Base: From arigo at codespeak.net Sat Dec 4 14:44:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 4 Dec 2010 14:44:09 +0100 (CET) Subject: [pypy-svn] r79802 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20101204134409.1E1E9282BE3@codespeak.net> Author: arigo Date: Sat Dec 4 14:44:08 2010 New Revision: 79802 Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Log: Add an assert (that never triggered so far). Modified: pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/ll2ctypes.py Sat Dec 4 14:44:08 2010 @@ -409,6 +409,7 @@ subcls = get_common_subclass(mixin_cls, instance.__class__) instance.__class__ = subcls instance._storage = ctypes_storage + assert ctypes_storage # null pointer? class _parentable_mixin(object): """Mixin added to _parentable containers when they become ctypes-based. From fijal at codespeak.net Sat Dec 4 14:49:02 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 4 Dec 2010 14:49:02 +0100 (CET) Subject: [pypy-svn] r79803 - pypy/branch/gc-huge-list Message-ID: <20101204134902.EE1A8282BE7@codespeak.net> Author: fijal Date: Sat Dec 4 14:49:01 2010 New Revision: 79803 Removed: pypy/branch/gc-huge-list/ Log: This was fixed by adding minimark gc From fijal at codespeak.net Sat Dec 4 14:51:01 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 4 Dec 2010 14:51:01 +0100 (CET) Subject: [pypy-svn] r79804 - pypy/branch/mapdict-without-jit Message-ID: <20101204135101.820DE36C1FD@codespeak.net> Author: fijal Date: Sat Dec 4 14:51:00 2010 New Revision: 79804 Removed: pypy/branch/mapdict-without-jit/ Log: Branch merged From fijal at codespeak.net Sat Dec 4 14:51:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 4 Dec 2010 14:51:53 +0100 (CET) Subject: [pypy-svn] r79805 - pypy/branch/asmgcc-64-old Message-ID: <20101204135153.ADEB8282BE9@codespeak.net> Author: fijal Date: Sat Dec 4 14:51:52 2010 New Revision: 79805 Removed: pypy/branch/asmgcc-64-old/ Log: Remove merged branch From fijal at codespeak.net Sat Dec 4 14:52:50 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 4 Dec 2010 14:52:50 +0100 (CET) Subject: [pypy-svn] r79806 - pypy/branch/jit-newx86 Message-ID: <20101204135250.4CC16282BEA@codespeak.net> Author: fijal Date: Sat Dec 4 14:52:47 2010 New Revision: 79806 Removed: pypy/branch/jit-newx86/ Log: Merged into x86_64 bti backend From fijal at codespeak.net Sat Dec 4 14:53:15 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 4 Dec 2010 14:53:15 +0100 (CET) Subject: [pypy-svn] r79807 - pypy/branch/x86-64-jit-backend Message-ID: <20101204135315.71C32282BEB@codespeak.net> Author: fijal Date: Sat Dec 4 14:53:13 2010 New Revision: 79807 Removed: pypy/branch/x86-64-jit-backend/ Log: Remove merged branch From fijal at codespeak.net Sat Dec 4 14:55:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 4 Dec 2010 14:55:53 +0100 (CET) Subject: [pypy-svn] r79808 - pypy/branch/remove-sys-recursionlimit Message-ID: <20101204135553.3D54C282BEF@codespeak.net> Author: fijal Date: Sat Dec 4 14:55:51 2010 New Revision: 79808 Added: pypy/branch/remove-sys-recursionlimit/ (props changed) - copied from r79807, pypy/trunk/ Log: A branch to remove functionality of raising RuntimeError when counter of recursion limit triggers. We have such functionality that measures stack size and is generally superior. From arigo at codespeak.net Sat Dec 4 15:04:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 4 Dec 2010 15:04:11 +0100 (CET) Subject: [pypy-svn] r79809 - pypy/extradoc/planning Message-ID: <20101204140411.64C9A282BF0@codespeak.net> Author: arigo Date: Sat Dec 4 15:04:09 2010 New Revision: 79809 Added: pypy/extradoc/planning/1.4.1.txt Log: A file listing differences between releases 1.4 and 1.4.1. Meant to also contain the planned work. Added: pypy/extradoc/planning/1.4.1.txt ============================================================================== --- (empty file) +++ pypy/extradoc/planning/1.4.1.txt Sat Dec 4 15:04:09 2010 @@ -0,0 +1,17 @@ +========================================== +Differences between releases 1.4 and 1.4.1 +========================================== + + +* In long-running processes, the assembler generated by old JIT-compilations + is now freed. There should be no more leak, however long the process runs. + +* Improve the performance of the ``binascii`` module, and of ``hashlib.md5`` + and ``hashlib.sha``. + +* Fix a corner case in the GC (minimark). + + +Plan: + +* ? From arigo at codespeak.net Sat Dec 4 15:19:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 4 Dec 2010 15:19:16 +0100 (CET) Subject: [pypy-svn] r79810 - pypy/trunk/pypy/module/array/test Message-ID: <20101204141916.9FE59282BF1@codespeak.net> Author: arigo Date: Sat Dec 4 15:19:15 2010 New Revision: 79810 Modified: pypy/trunk/pypy/module/array/test/test_array.py Log: CPython always says that Array('I') contains longs, but on 64-bits we say that it contains ints. This is a difference that is probably ok. Modified: pypy/trunk/pypy/module/array/test/test_array.py ============================================================================== --- pypy/trunk/pypy/module/array/test/test_array.py (original) +++ pypy/trunk/pypy/module/array/test/test_array.py Sat Dec 4 15:19:15 2010 @@ -64,6 +64,7 @@ raises(TypeError, self.array, tc, None) def test_value_range(self): + import sys values = (-129, 128, -128, 127, 0, 255, -1, 256, -32768, 32767, -32769, 32768, 65535, 65536, -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, @@ -88,7 +89,12 @@ a.append(v) for i, v in enumerate(ok * 2): assert a[i] == v - assert type(a[i]) is pt + assert type(a[i]) is pt or ( + # A special case: we return ints in Array('I') on 64-bits, + # whereas CPython returns longs. The difference is + # probably acceptable. + tc == 'I' and + sys.maxint > 2147483647 and type(a[i]) is int) for v in ok: a[1] = v assert a[0] == ok[0] From david at codespeak.net Sat Dec 4 15:19:45 2010 From: david at codespeak.net (david at codespeak.net) Date: Sat, 4 Dec 2010 15:19:45 +0100 (CET) Subject: [pypy-svn] r79811 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101204141945.DA4DB282BF4@codespeak.net> Author: david Date: Sat Dec 4 15:19:44 2010 New Revision: 79811 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: Refactor guard_class code a bit Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Sat Dec 4 15:19:44 2010 @@ -268,46 +268,45 @@ # from ../x86/assembler.py:1265 def emit_op_guard_class(self, op, regalloc, fcond): - locs, boxes = self._prepare_guard_class(op, regalloc, fcond) + locs = self._prepare_guard_class(op, regalloc, fcond) self._cmp_guard_class(op, locs, regalloc, fcond) - regalloc.possibly_free_vars(boxes) return fcond def emit_op_guard_nonnull_class(self, op, regalloc, fcond): - locs, boxes = self._prepare_guard_class(op, regalloc, fcond) + locs = self._prepare_guard_class(op, regalloc, fcond) self.mc.CMP_ri(locs[0].value, 0) self._emit_guard(op, regalloc, c.NE) self._cmp_guard_class(op, locs, regalloc, fcond) - regalloc.possibly_free_vars(boxes) return fcond def _prepare_guard_class(self, op, regalloc, fcond): assert isinstance(op.getarg(0), Box) + boxes = list(op.getarglist()) + + x, x_box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) + boxes.append(x_box) + + t = TempBox() + y = regalloc.force_allocate_reg(t, boxes) + boxes.append(t) + y_val = op.getarg(1).getint() + self.mc.gen_load_int(y.value, rffi.cast(lltype.Signed, y_val)) - x = regalloc.make_sure_var_in_reg(op.getarg(0), imm_fine=False) - y_temp = TempBox() - y = regalloc.force_allocate_reg(y_temp) - arg = op.getarg(1).getint() - self.mc.gen_load_int(y.value, rffi.cast(lltype.Signed, arg)) - return [x, y], [op.getarg(0), y_temp] + regalloc.possibly_free_vars(boxes) + return [x, y] def _cmp_guard_class(self, op, locs, regalloc, fcond): offset = self.cpu.vtable_offset - x = locs[0] - y = locs[1] + x, y = locs if offset is not None: - t0 = TempBox() - l0 = regalloc.force_allocate_reg(t0) assert offset == 0 - self.mc.LDR_ri(l0.value, x.value, offset) - self.mc.CMP_rr(l0.value, y.value) - regalloc.possibly_free_var(t0) + self.mc.LDR_ri(r.ip.value, x.value, offset) + self.mc.CMP_rr(r.ip.value, y.value) else: raise NotImplementedError # XXX port from x86 backend once gc support is in place - regalloc.possibly_free_vars_for_op(op) return self._emit_guard(op, regalloc, c.EQ) @@ -346,7 +345,6 @@ return cond def _emit_call(self, adr, args, regalloc, fcond=c.AL, save_all_regs=False, result=None): - locs = [] # all arguments past the 4th go on the stack # XXX support types other than int (one word types) n = 0 @@ -365,7 +363,6 @@ for i in range(0, reg_args): l = regalloc.make_sure_var_in_reg(args[i], selected_reg=r.all_regs[i]) - locs.append(l) # XXX use PUSH here instead of spilling every reg for itself regalloc.before_call(save_all_regs=save_all_regs) @@ -822,7 +819,7 @@ boxes = list(op.getarglist()) base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) - boxes = [box] + boxes.append(box) ofs_loc, box = self._ensure_value_is_boxed(boxes[1], regalloc, boxes) boxes.append(box) @@ -1024,7 +1021,7 @@ arglocs = [] for arg in args: t = TempBox() - l = regalloc.force_allocate_reg(t) + l = regalloc.force_allocate_reg(t, arglocs) self.mc.gen_load_int(l.value, arg) arglocs.append(t) return arglocs @@ -1061,6 +1058,7 @@ self._emit_call(self.malloc_func_addr, arglocs, regalloc, result=op.result) regalloc.possibly_free_vars(arglocs) + regalloc.possibly_free_var(op.result) return fcond def emit_op_new_with_vtable(self, op, regalloc, fcond): @@ -1070,6 +1068,7 @@ self._emit_call(self.malloc_func_addr, arglocs, regalloc, result=op.result) regalloc.possibly_free_vars(arglocs) + regalloc.possibly_free_var(op.result) self.set_vtable(op.result, op.getarg(0), regalloc) return fcond From arigo at codespeak.net Sat Dec 4 15:33:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 4 Dec 2010 15:33:15 +0100 (CET) Subject: [pypy-svn] r79812 - pypy/trunk/pypy/module/pypyjit/test Message-ID: <20101204143315.6E9EF282BF1@codespeak.net> Author: arigo Date: Sat Dec 4 15:33:13 2010 New Revision: 79812 Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Log: Fix test_pypy_c for arrays on 64 bits. Modified: pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/trunk/pypy/module/pypyjit/test/test_pypy_c.py Sat Dec 4 15:33:13 2010 @@ -885,11 +885,14 @@ def test_array_sum(self): for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): res = 19352859 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array @@ -937,11 +940,14 @@ print '='*65 print '='*20, 'running test for tc=%r' % (tc,), '='*20 res = 73574560 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array From arigo at codespeak.net Sat Dec 4 15:39:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 4 Dec 2010 15:39:11 +0100 (CET) Subject: [pypy-svn] r79813 - pypy/branch/inline-shadowstack Message-ID: <20101204143911.57527282BF4@codespeak.net> Author: arigo Date: Sat Dec 4 15:39:09 2010 New Revision: 79813 Added: pypy/branch/inline-shadowstack/ - copied from r79812, pypy/trunk/ Log: A branch in which to improve shadow tracking. From fijal at codespeak.net Sat Dec 4 15:41:15 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 4 Dec 2010 15:41:15 +0100 (CET) Subject: [pypy-svn] r79814 - pypy/extradoc/planning Message-ID: <20101204144115.73E4B282BF5@codespeak.net> Author: fijal Date: Sat Dec 4 15:41:13 2010 New Revision: 79814 Modified: pypy/extradoc/planning/1.4.1.txt Log: plans (?) Modified: pypy/extradoc/planning/1.4.1.txt ============================================================================== --- pypy/extradoc/planning/1.4.1.txt (original) +++ pypy/extradoc/planning/1.4.1.txt Sat Dec 4 15:41:13 2010 @@ -14,4 +14,10 @@ Plan: -* ? +* Fix bug in optimizeopt + +* Out of line guards (?) + +* Merge jit-unroll-loops (?) + +* Migrate to mercurial From fijal at codespeak.net Sat Dec 4 16:13:57 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sat, 4 Dec 2010 16:13:57 +0100 (CET) Subject: [pypy-svn] r79815 - in pypy/branch/remove-sys-recursionlimit/pypy: interpreter module/_stackless module/sys Message-ID: <20101204151357.88C93282BD4@codespeak.net> Author: fijal Date: Sat Dec 4 16:13:55 2010 New Revision: 79815 Modified: pypy/branch/remove-sys-recursionlimit/pypy/interpreter/executioncontext.py pypy/branch/remove-sys-recursionlimit/pypy/module/_stackless/interp_coroutine.py pypy/branch/remove-sys-recursionlimit/pypy/module/sys/vm.py Log: remove tracking of recursion depth. issue a deprecation warning when calling sys.setrecursionlimit Modified: pypy/branch/remove-sys-recursionlimit/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/remove-sys-recursionlimit/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/remove-sys-recursionlimit/pypy/interpreter/executioncontext.py Sat Dec 4 16:13:55 2010 @@ -27,7 +27,6 @@ def __init__(self, space): self.space = space self.topframeref = jit.vref_None - self.framestackdepth = 0 # tracing: space.frame_trace_action.fire() must be called to ensure # that tracing occurs whenever self.w_tracefunc or self.is_tracing # is modified. @@ -54,9 +53,6 @@ return frame def enter(self, frame): - if self.framestackdepth > self.space.sys.recursionlimit: - raise self.space.prebuilt_recursion_error - self.framestackdepth += 1 frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) @@ -66,7 +62,6 @@ self._trace(frame, 'leaveframe', self.space.w_None) finally: self.topframeref = frame.f_backref - self.framestackdepth -= 1 jit.virtual_ref_finish(frame) if self.w_tracefunc is not None and not frame.hide(): @@ -80,7 +75,6 @@ def __init__(self): self.topframe = None - self.framestackdepth = 0 self.w_tracefunc = None self.profilefunc = None self.w_profilefuncarg = None @@ -88,7 +82,6 @@ def enter(self, ec): ec.topframeref = jit.non_virtual_ref(self.topframe) - ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc ec.w_profilefuncarg = self.w_profilefuncarg @@ -97,7 +90,6 @@ def leave(self, ec): self.topframe = ec.gettopframe() - self.framestackdepth = ec.framestackdepth self.w_tracefunc = ec.w_tracefunc self.profilefunc = ec.profilefunc self.w_profilefuncarg = ec.w_profilefuncarg @@ -105,7 +97,6 @@ def clear_framestack(self): self.topframe = None - self.framestackdepth = 0 # the following interface is for pickling and unpickling def getstate(self, space): @@ -121,17 +112,14 @@ self.topframe = space.interp_w(PyFrame, frames_w[-1]) else: self.topframe = None - self.framestackdepth = len(frames_w) def getframestack(self): - index = self.framestackdepth - lst = [None] * index + lst = [] f = self.topframe - while index > 0: - index -= 1 - lst[index] = f + while f is not None: + lst.append(f) f = f.f_backref() - assert f is None + lst.reverse() return lst # coroutine: I think this is all, folks! Modified: pypy/branch/remove-sys-recursionlimit/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/remove-sys-recursionlimit/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/remove-sys-recursionlimit/pypy/module/_stackless/interp_coroutine.py Sat Dec 4 16:13:55 2010 @@ -304,16 +304,14 @@ def w_descr__framestack(space, self): assert isinstance(self, AppCoroutine) - index = self.subctx.framestackdepth - if not index: - return space.newtuple([]) - items = [None] * index f = self.subctx.topframe - while index > 0: - index -= 1 - items[index] = space.wrap(f) + items = [] + if not f: + return space.newtuple([]) + while f is not None: + items.append(space.wrap(f)) f = f.f_backref() - assert f is None + items.reverse() return space.newtuple(items) def makeStaticMethod(module, classname, funcname): Modified: pypy/branch/remove-sys-recursionlimit/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/remove-sys-recursionlimit/pypy/module/sys/vm.py (original) +++ pypy/branch/remove-sys-recursionlimit/pypy/module/sys/vm.py Sat Dec 4 16:13:55 2010 @@ -41,27 +41,21 @@ f = ec.getnextframe_nohidden(f) return space.wrap(f) -# directly from the C code in ceval.c, might be moved somewhere else. - def setrecursionlimit(space, w_new_limit): - """Set the maximum depth of the Python interpreter stack to n. This -limit prevents infinite recursion from causing an overflow of the C -stack and crashing Python. The highest possible limit is platform -dependent.""" + """DEPRECATED on PyPy. Will issue warning and not work + """ new_limit = space.int_w(w_new_limit) if new_limit <= 0: raise OperationError(space.w_ValueError, space.wrap("recursion limit must be positive")) # global recursion_limit # we need to do it without writing globals. + space.warn('setrecursionlimit deprecated', space.w_DeprecationWarning) space.sys.recursionlimit = new_limit def getrecursionlimit(space): - """Return the current value of the recursion limit, the maximum depth - of the Python interpreter stack. This limit prevents infinite - recursion from causing an overflow of the C stack and crashing Python. + """DEPRECATED on PyPy. Will issue warning and not work """ - return space.wrap(space.sys.recursionlimit) def setcheckinterval(space, interval): From hakanardo at codespeak.net Sat Dec 4 16:41:37 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 4 Dec 2010 16:41:37 +0100 (CET) Subject: [pypy-svn] r79816 - in pypy/branch/jit-unroll-loops/pypy/jit/metainterp: optimizeopt test Message-ID: <20101204154137.034A3282BAD@codespeak.net> Author: hakanardo Date: Sat Dec 4 16:41:34 2010 New Revision: 79816 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/rewrite.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py Log: We now need to optimize GUARD_NONNULL_CLASS since they might be inserted while inlining Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/rewrite.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/rewrite.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/rewrite.py Sat Dec 4 16:41:34 2010 @@ -255,6 +255,10 @@ last_guard_index = value.last_guard_index value.make_constant_class(expectedclassbox, last_guard_index) + def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False): + self.optimize_GUARD_NONNULL(op, True) + self.optimize_GUARD_CLASS(op, dryrun) + def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False): if dryrun: return if not self.optimizer.exception_might_have_happened: Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py Sat Dec 4 16:41:34 2010 @@ -493,6 +493,42 @@ """ self.optimize_loop(ops, expected, preamble) + def test_guard_nonnull_class_1(self): + ops = """ + [p0] + guard_class(p0, ConstClass(node_vtable)) [] + guard_nonnull(p0) [] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] + jump(p0) + """ + preamble = """ + [p0] + guard_class(p0, ConstClass(node_vtable)) [] + jump(p0) + """ + expected = """ + [p0] + jump(p0) + """ + self.optimize_loop(ops, expected, preamble) + + def test_guard_nonnull_class_2(self): + ops = """ + [p0] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] + jump(p0) + """ + preamble = """ + [p0] + guard_nonnull_class(p0, ConstClass(node_vtable)) [] + jump(p0) + """ + expected = """ + [p0] + jump(p0) + """ + self.optimize_loop(ops, expected, preamble) + def test_int_is_true_1(self): ops = """ [i0] From david at codespeak.net Sat Dec 4 16:46:09 2010 From: david at codespeak.net (david at codespeak.net) Date: Sat, 4 Dec 2010 16:46:09 +0100 (CET) Subject: [pypy-svn] r79817 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . helper test Message-ID: <20101204154609.618B3282BAD@codespeak.net> Author: david Date: Sat Dec 4 16:46:07 2010 New Revision: 79817 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Log: Different register allocation changes Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py Sat Dec 4 16:46:07 2010 @@ -52,16 +52,17 @@ return f def gen_emit_op_by_helper_call(opname): + helper = getattr(AbstractARMv7Builder, opname) def f(self, op, regalloc, fcond): assert fcond is not None a0 = op.getarg(0) a1 = op.getarg(1) - arg1 = regalloc.make_sure_var_in_reg(a0, selected_reg=r.r0, imm_fine=False) - arg2 = regalloc.make_sure_var_in_reg(a1, selected_reg=r.r1, imm_fine=False) + arg1 = regalloc.make_sure_var_in_reg(a0, selected_reg=r.r0) + arg2 = regalloc.make_sure_var_in_reg(a1, selected_reg=r.r1) assert arg1 == r.r0 assert arg2 == r.r1 regalloc.before_call() - getattr(self.mc, opname)(fcond) + helper(self.mc, fcond) regalloc.after_call(op.result) regalloc.possibly_free_var(a0) Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Sat Dec 4 16:46:07 2010 @@ -30,9 +30,8 @@ def emit_op_int_add(self, op, regalloc, fcond): #XXX check if neg values are supported for imm values - a0 = op.getarg(0) - a1 = op.getarg(1) boxes = list(op.getarglist()) + a0, a1 = boxes imm_a0 = self._check_imm_arg(a0) imm_a1 = self._check_imm_arg(a1) if not imm_a0 and imm_a1: @@ -63,9 +62,8 @@ def emit_op_int_sub(self, op, regalloc, fcond): #XXX check if neg values are supported for imm values - a0 = op.getarg(0) - a1 = op.getarg(1) - boxes = [a0, a1] + boxes = list(op.getarglist()) + a0, a1 = boxes imm_a0 = self._check_imm_arg(a0) imm_a1 = self._check_imm_arg(a1) if not imm_a0 and imm_a1: @@ -99,9 +97,8 @@ return fcond def emit_op_int_mul(self, op, regalloc, fcond): - a0 = op.getarg(0) - a1 = op.getarg(1) boxes = list(op.getarglist()) + a0, a1 = boxes reg1, box = self._ensure_value_is_boxed(a0, regalloc, forbidden_vars=boxes) boxes.append(box) @@ -116,9 +113,8 @@ #ref: http://blogs.arm.com/software-enablement/detecting-overflow-from-mul/ def emit_guard_int_mul_ovf(self, op, guard, regalloc, fcond): - a0 = op.getarg(0) - a1 = op.getarg(1) boxes = list(op.getarglist()) + a0, a1 = boxes reg1, box = self._ensure_value_is_boxed(a0, regalloc, forbidden_vars=boxes) boxes.append(box) @@ -178,24 +174,22 @@ emit_op_int_is_zero = gen_emit_op_unary_cmp(c.EQ, c.NE) def emit_op_int_invert(self, op, regalloc, fcond): - a0 = op.getarg(0) - reg = regalloc.make_sure_var_in_reg(a0, imm_fine=False) - res = regalloc.force_allocate_reg(op.result, [a0]) + reg, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) + res = regalloc.force_allocate_reg(op.result, [box]) + regalloc.possibly_free_var(box) + regalloc.possibly_free_var(op.result) self.mc.MVN_rr(res.value, reg.value) - regalloc.possibly_free_vars_for_op(op) - regalloc.possibly_free_var(op.result) return fcond #XXX check for a better way of doing this def emit_op_int_neg(self, op, regalloc, fcond): - arg = op.getarg(0) - resbox = op.result - l0 = regalloc.make_sure_var_in_reg(arg) + l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) + resloc = regalloc.force_allocate_reg(op.result, [box]) + regalloc.possibly_free_vars([box, op.result]) + self.mc.MVN_ri(r.ip.value, imm=~-1) - resloc = regalloc.force_allocate_reg(resbox, [arg]) self.mc.MUL(resloc.value, l0.value, r.ip.value) - regalloc.possibly_free_vars([arg, resbox]) return fcond class GuardOpAssembler(object): @@ -224,17 +218,15 @@ return c.AL def emit_op_guard_true(self, op, regalloc, fcond): - a0 = op.getarg(0) - l0 = regalloc.make_sure_var_in_reg(a0, imm_fine=False) + l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) + regalloc.possibly_free_var(box) self.mc.CMP_ri(l0.value, 0) - regalloc.possibly_free_var(a0) return self._emit_guard(op, regalloc, c.NE) def emit_op_guard_false(self, op, regalloc, fcond): - a0 = op.getarg(0) - l0 = regalloc.make_sure_var_in_reg(a0, imm_fine=False) + l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) + regalloc.possibly_free_var(box) self.mc.CMP_ri(l0.value, 0) - regalloc.possibly_free_var(a0) return self._emit_guard(op, regalloc, c.EQ) def emit_op_guard_value(self, op, regalloc, fcond): @@ -346,7 +338,6 @@ def _emit_call(self, adr, args, regalloc, fcond=c.AL, save_all_regs=False, result=None): # all arguments past the 4th go on the stack - # XXX support types other than int (one word types) n = 0 n_args = len(args) if n_args > 4: @@ -354,9 +345,9 @@ n = stack_args*WORD self._adjust_sp(n, regalloc, fcond=fcond) for i in range(4, n_args): - reg = regalloc.make_sure_var_in_reg(args[i]) + reg, box = self._ensure_value_is_boxed(args[i], regalloc) self.mc.STR_ri(reg.value, r.sp.value, (i-4)*WORD) - regalloc.possibly_free_var(args[i]) + regalloc.possibly_free_var(box) reg_args = min(n_args, 4) @@ -365,6 +356,7 @@ selected_reg=r.all_regs[i]) # XXX use PUSH here instead of spilling every reg for itself regalloc.before_call(save_all_regs=save_all_regs) + regalloc.possibly_free_vars(args) self.mc.BL(adr) @@ -374,7 +366,6 @@ if n_args > 4: assert n > 0 self._adjust_sp(-n, regalloc, fcond=fcond) - regalloc.possibly_free_vars(args) return fcond def emit_op_same_as(self, op, regalloc, fcond): @@ -1036,8 +1027,7 @@ regalloc, result=res_v) loc = regalloc.make_sure_var_in_reg(v, [res_v]) regalloc.possibly_free_var(v) - if tempbox is not None: - regalloc.possibly_free_var(tempbox) + regalloc.possibly_free_var(tempbox) # XXX combine with emit_op_setfield_gc operation base_loc = regalloc.loc(res_v) @@ -1067,22 +1057,21 @@ arglocs = self._prepare_args_for_new_op(descrsize, regalloc) self._emit_call(self.malloc_func_addr, arglocs, regalloc, result=op.result) + resloc = regalloc.loc(op.result) # r0 + self.set_vtable(resloc, classint, regalloc) regalloc.possibly_free_vars(arglocs) regalloc.possibly_free_var(op.result) - self.set_vtable(op.result, op.getarg(0), regalloc) return fcond - def set_vtable(self, result, vtable, regalloc): - loc = regalloc.loc(result) + def set_vtable(self, loc, vtable, regalloc): if self.cpu.vtable_offset is not None: assert loc.is_reg() - adr = rffi.cast(lltype.Signed, vtable.getint()) + adr = rffi.cast(lltype.Signed, vtable) t = TempBox() loc_vtable = regalloc.force_allocate_reg(t) + regalloc.possibly_free_var(t) self.mc.gen_load_int(loc_vtable.value, adr) - #assert isinstance(loc_vtable, ImmedLoc) self.mc.STR_ri(loc_vtable.value, loc.value, self.cpu.vtable_offset) - regalloc.possibly_free_var(t) def emit_op_new_array(self, op, regalloc, fcond): gc_ll_descr = self.cpu.gc_ll_descr Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Sat Dec 4 16:46:07 2010 @@ -31,13 +31,13 @@ OPERATIONS.append(test_ll_random.UnicodeLenOperation(rop.UNICODELEN)) # for i in range(2): - OPERATIONS.append(test_ll_random.GuardClassOperation(rop.GUARD_CLASS)) - #OPERATIONS.append(test_ll_random.CallOperation(rop.CALL)) - #OPERATIONS.append(test_ll_random.RaisingCallOperation(rop.CALL)) - #OPERATIONS.append(test_ll_random.RaisingCallOperationGuardNoException(rop.CALL)) - #OPERATIONS.append(test_ll_random.RaisingCallOperationWrongGuardException(rop.CALL)) - #OPERATIONS.append(test_ll_random.CallOperationException(rop.CALL)) -OPERATIONS.append(test_ll_random.GuardNonNullClassOperation(rop.GUARD_NONNULL_CLASS)) +# OPERATIONS.append(test_ll_random.GuardClassOperation(rop.GUARD_CLASS)) + OPERATIONS.append(test_ll_random.CallOperation(rop.CALL)) + OPERATIONS.append(test_ll_random.RaisingCallOperation(rop.CALL)) + OPERATIONS.append(test_ll_random.RaisingCallOperationGuardNoException(rop.CALL)) + OPERATIONS.append(test_ll_random.RaisingCallOperationWrongGuardException(rop.CALL)) + OPERATIONS.append(test_ll_random.CallOperationException(rop.CALL)) +#OPERATIONS.append(test_ll_random.GuardNonNullClassOperation(rop.GUARD_NONNULL_CLASS)) From danchr at codespeak.net Sat Dec 4 20:44:55 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 20:44:55 +0100 (CET) Subject: [pypy-svn] r79818 - pypy/branch/cpyext-darwin Message-ID: <20101204194455.02F1E282B9C@codespeak.net> Author: danchr Date: Sat Dec 4 20:44:53 2010 New Revision: 79818 Added: pypy/branch/cpyext-darwin/ (props changed) - copied from r79817, pypy/trunk/ Log: Create a branch for my cpyext hackery. From danchr at codespeak.net Sat Dec 4 20:50:54 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 20:50:54 +0100 (CET) Subject: [pypy-svn] r79819 - pypy/branch/cpyext-darwin/pypy/translator/platform Message-ID: <20101204195054.ADB7B282BAD@codespeak.net> Author: danchr Date: Sat Dec 4 20:50:53 2010 New Revision: 79819 Modified: pypy/branch/cpyext-darwin/pypy/translator/platform/posix.py Log: Fix building --shared on Mac OS X, when non-default architecture is used. Modified: pypy/branch/cpyext-darwin/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/translator/platform/posix.py (original) +++ pypy/branch/cpyext-darwin/pypy/translator/platform/posix.py Sat Dec 4 20:50:53 2010 @@ -166,7 +166,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'], - '$(CC_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') + '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') return m From danchr at codespeak.net Sat Dec 4 20:50:59 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 20:50:59 +0100 (CET) Subject: [pypy-svn] r79820 - pypy/branch/cpyext-darwin/pypy/tool/release Message-ID: <20101204195059.39930282BE3@codespeak.net> Author: danchr Date: Sat Dec 4 20:50:57 2010 New Revision: 79820 Modified: pypy/branch/cpyext-darwin/pypy/tool/release/package.py (contents, props changed) Log: Don't strip exported symbols in package.py Doing so breaks cpyext on Mac OS X. Modified: pypy/branch/cpyext-darwin/pypy/tool/release/package.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/tool/release/package.py (original) +++ pypy/branch/cpyext-darwin/pypy/tool/release/package.py Sat Dec 4 20:50:57 2010 @@ -78,7 +78,7 @@ old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) # ignore errors + os.system("strip -x " + str(archive_pypy_c)) # ignore errors if USE_TARFILE_MODULE: import tarfile tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') From danchr at codespeak.net Sat Dec 4 20:51:01 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 20:51:01 +0100 (CET) Subject: [pypy-svn] r79821 - pypy/branch/cpyext-darwin/lib-python/modified-2.5.2/distutils Message-ID: <20101204195101.4EE7B282BDD@codespeak.net> Author: danchr Date: Sat Dec 4 20:50:59 2010 New Revision: 79821 Modified: pypy/branch/cpyext-darwin/lib-python/modified-2.5.2/distutils/unixccompiler.py Log: distutils: Add hack to allow building C extensions on Mac OS X. Modified: pypy/branch/cpyext-darwin/lib-python/modified-2.5.2/distutils/unixccompiler.py ============================================================================== --- pypy/branch/cpyext-darwin/lib-python/modified-2.5.2/distutils/unixccompiler.py (original) +++ pypy/branch/cpyext-darwin/lib-python/modified-2.5.2/distutils/unixccompiler.py Sat Dec 4 20:50:59 2010 @@ -121,7 +121,22 @@ } if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() executables['ranlib'] = ["ranlib"] + executables['linker_so'] += ['-undefined', 'dynamic_lookup'] + + for k, v in executables.iteritems(): + if v and v[0] == 'cc': + v += ['-arch', arch] + # Needed for the filename generation methods provided by the base # class, CCompiler. NB. whoever instantiates/uses a particular From danchr at codespeak.net Sat Dec 4 20:51:07 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 20:51:07 +0100 (CET) Subject: [pypy-svn] r79822 - in pypy/branch/cpyext-darwin/pypy/module/cpyext: . test Message-ID: <20101204195107.493E0282BE9@codespeak.net> Author: danchr Date: Sat Dec 4 20:51:05 2010 New Revision: 79822 Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/pyerrors.py pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_pyerrors.py Log: Implement PyErr_SetFromErrnoWithFilename() Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/pyerrors.py Sat Dec 4 20:51:05 2010 @@ -129,10 +129,29 @@ function around a system call can write return PyErr_SetFromErrno(type); when the system call returns an error. Return value: always NULL.""" + PyErr_SetFromErrnoWithFilename(space, w_type, + lltype.nullptr(rffi.CCHARP.TO)) + + at cpython_api([PyObject, rffi.CCHARP], PyObject) +def PyErr_SetFromErrnoWithFilename(space, w_type, llfilename): + """Similar to PyErr_SetFromErrno(), with the additional behavior that if + filename is not NULL, it is passed to the constructor of type as a third + parameter. In the case of exceptions such as IOError and OSError, + this is used to define the filename attribute of the exception instance. + Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() msg = os.strerror(errno) - w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + if llfilename: + w_filename = rffi.charp2str(llfilename) + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg), + space.wrap(w_filename)) + else: + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg)) raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py Sat Dec 4 20:51:05 2010 @@ -668,15 +668,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], PyObject) -def PyErr_SetFromErrnoWithFilename(space, type, filename): - """Similar to PyErr_SetFromErrno(), with the additional behavior that if - filename is not NULL, it is passed to the constructor of type as a third - parameter. In the case of exceptions such as IOError and OSError, - this is used to define the filename attribute of the exception instance. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([rffi.INT_real], PyObject) def PyErr_SetFromWindowsErr(space, ierr): """This is a convenience function to raise WindowsError. If called with Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_pyerrors.py Sat Dec 4 20:51:05 2010 @@ -185,3 +185,26 @@ except OSError, e: assert e.errno == errno.EBADF assert e.strerror == os.strerror(errno.EBADF) + assert e.filename == None + + def test_SetFromErrnoWithFilename(self): + import sys + if sys.platform != 'win32': + skip("callbacks through ll2ctypes modify errno") + import errno, os + + module = self.import_extension('foo', [ + ("set_from_errno", "METH_NOARGS", + ''' + errno = EBADF; + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "blyf"); + return NULL; + '''), + ], + prologue="#include ") + try: + module.set_from_errno() + except OSError, e: + assert e.filename == "blyf" + assert e.errno == errno.EBADF + assert e.strerror == os.strerror(errno.EBADF) From danchr at codespeak.net Sat Dec 4 20:51:11 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 20:51:11 +0100 (CET) Subject: [pypy-svn] r79823 - in pypy/branch/cpyext-darwin/pypy/module/cpyext: . test Message-ID: <20101204195111.BE7C7282BEF@codespeak.net> Author: danchr Date: Sat Dec 4 20:51:08 2010 New Revision: 79823 Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/object.py pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_object.py Log: Implement PyFile_Check & PyFile_CheckExact Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/object.py Sat Dec 4 20:51:08 2010 @@ -2,12 +2,13 @@ from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, - Py_GE, CONST_STRING, FILEP, fwrite) + Py_GE, CONST_STRING, FILEP, fwrite, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference, get_typedescr, RefcountState) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall +from pypy.module._file.interp_file import W_File from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject from pypy.interpreter.error import OperationError @@ -428,6 +429,8 @@ rffi.free_nonmovingbuffer(data, buf) return 0 +PyFile_Check, PyFile_CheckExact = build_type_checkers("File", W_File) + @cpython_api([CONST_STRING, CONST_STRING], PyObject) def PyFile_FromString(space, filename, mode): """ @@ -437,4 +440,3 @@ w_filename = space.wrap(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) - Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py Sat Dec 4 20:51:08 2010 @@ -791,21 +791,6 @@ successful invocation of Py_EnterRecursiveCall().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_Check(space, p): - """Return true if its argument is a PyFileObject or a subtype of - PyFileObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_CheckExact(space, p): - """Return true if its argument is a PyFileObject, but not a subtype of - PyFileObject. - """ - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_object.py Sat Dec 4 20:51:08 2010 @@ -188,6 +188,10 @@ rffi.free_charp(filename) rffi.free_charp(mode) + assert api.PyFile_Check(w_file) + assert api.PyFile_CheckExact(w_file) + assert not api.PyFile_Check(space.wrap("text")) + space.call_method(w_file, "write", space.wrap("text")) space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" From danchr at codespeak.net Sat Dec 4 20:51:13 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 20:51:13 +0100 (CET) Subject: [pypy-svn] r79824 - in pypy/branch/cpyext-darwin/pypy/module/cpyext: . test Message-ID: <20101204195113.8227B282BEA@codespeak.net> Author: danchr Date: Sat Dec 4 20:51:11 2010 New Revision: 79824 Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/object.py pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_object.py Log: Implement PyFile_GetLine() Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/object.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/object.py Sat Dec 4 20:51:11 2010 @@ -431,6 +431,32 @@ PyFile_Check, PyFile_CheckExact = build_type_checkers("File", W_File) + at cpython_api([PyObject, rffi.INT_real], PyObject) +def PyFile_GetLine(space, w_obj, n): + """ + Equivalent to p.readline([n]), this function reads one line from the + object p. p may be a file object or any object with a readline() + method. If n is 0, exactly one line is read, regardless of the length of + the line. If n is greater than 0, no more than n bytes will be read + from the file; a partial line can be returned. In both cases, an empty string + is returned if the end of the file is reached immediately. If n is less than + 0, however, one line is read regardless of length, but EOFError is + raised if the end of the file is reached immediately.""" + try: + w_readline = space.getattr(w_obj, space.wrap('readline')) + except OperationError: + raise OperationError( + space.w_TypeError, space.wrap( + "argument must be a file, or have a readline() method.")) + + n = rffi.cast(lltype.Signed, n) + if space.is_true(space.gt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline, space.wrap(n)) + elif space.is_true(space.lt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline) + else: + # XXX Raise EOFError as specified + return space.call_function(w_readline) @cpython_api([CONST_STRING, CONST_STRING], PyObject) def PyFile_FromString(space, filename, mode): """ Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/stubs.py Sat Dec 4 20:51:11 2010 @@ -824,22 +824,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], PyObject) -def PyFile_GetLine(space, p, n): - """ - - - - Equivalent to p.readline([n]), this function reads one line from the - object p. p may be a file object or any object with a readline() - method. If n is 0, exactly one line is read, regardless of the length of - the line. If n is greater than 0, no more than n bytes will be read - from the file; a partial line can be returned. In both cases, an empty string - is returned if the end of the file is reached immediately. If n is less than - 0, however, one line is read regardless of length, but EOFError is - raised if the end of the file is reached immediately.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyFile_Name(space, p): """Return the name of the file specified by p as a string object.""" Modified: pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/cpyext-darwin/pypy/module/cpyext/test/test_object.py Sat Dec 4 20:51:11 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ - Py_GE, Py_GT + Py_GE, Py_GT, fopen, fclose, fwrite from pypy.tool.udir import udir class TestObject(BaseApiTest): @@ -196,6 +196,37 @@ space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" + def test_file_getline(self, space, api): + filename = rffi.str2charp(str(udir / "_test_file")) + + mode = rffi.str2charp("w") + w_file = api.PyFile_FromString(filename, mode) + space.call_method(w_file, "write", + space.wrap("line1\nline2\nline3\nline4")) + space.call_method(w_file, "close") + + rffi.free_charp(mode) + mode = rffi.str2charp("r") + w_file = api.PyFile_FromString(filename, mode) + rffi.free_charp(filename) + rffi.free_charp(mode) + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "line1\n" + + w_line = api.PyFile_GetLine(w_file, 4) + assert space.str_w(w_line) == "line" + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "2\n" + + # XXX We ought to raise an EOFError here, but don't + w_line = api.PyFile_GetLine(w_file, -1) + # assert api.PyErr_Occurred() is space.w_EOFError + assert space.str_w(w_line) == "line3\n" + + space.call_method(w_file, "close") + class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) From david at codespeak.net Sat Dec 4 21:25:40 2010 From: david at codespeak.net (david at codespeak.net) Date: Sat, 4 Dec 2010 21:25:40 +0100 (CET) Subject: [pypy-svn] r79825 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101204202540.89425282BAD@codespeak.net> Author: david Date: Sat Dec 4 21:25:38 2010 New Revision: 79825 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Log: Update str and unicode copy ops to work better with const values Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Sat Dec 4 21:25:38 2010 @@ -450,6 +450,8 @@ return False def _ensure_value_is_boxed(self, thing, regalloc, forbidden_vars=[]): + # XXX create TempBox subclasses with the corresponding type flags and + # remove the overridden force_allocate_reg once done box = None loc = None if isinstance(thing, Const): Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Sat Dec 4 21:25:38 2010 @@ -687,48 +687,47 @@ def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): # compute the source address - args = op.getarglist() - boxes = [] - base_loc, box = self._ensure_value_is_boxed(args[0], regalloc) - boxes.append(box) - ofs_loc, box = self._ensure_value_is_boxed(args[2], regalloc) - boxes.append(box) + args = list(op.getarglist()) + base_loc, box = self._ensure_value_is_boxed(args[0], regalloc, args) + args.append(box) + ofs_loc, box = self._ensure_value_is_boxed(args[2], regalloc, args) + args.append(box) assert args[0] is not args[1] # forbidden case of aliasing - + regalloc.possibly_free_var(args[0]) + if args[3] is not args[2] is not args[4]: # MESS MESS MESS: don't free + regalloc.possibly_free_var(args[2]) # it if ==args[3] or args[4] srcaddr_box = TempBox() forbidden_vars = [args[1], args[3], args[4], srcaddr_box] srcaddr_loc = regalloc.force_allocate_reg(srcaddr_box, forbidden_vars) self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, is_unicode=is_unicode) - regalloc.possibly_free_var(boxes[0]) - if args[3] is not args[2] is not args[4]: # MESS MESS MESS: don't free - regalloc.possibly_free_var(boxes[1]) # it if ==args[3] or args[4] # compute the destination address - base_loc, box = self._ensure_value_is_boxed(args[1], regalloc) - boxes.append(box) - ofs_loc, box = self._ensure_value_is_boxed(args[3], regalloc) - boxes.append(box) + base_loc, box = self._ensure_value_is_boxed(args[1], regalloc, forbidden_vars) + args.append(box) + ofs_loc, box = self._ensure_value_is_boxed(args[3], regalloc, forbidden_vars) + args.append(box) assert base_loc.is_reg() assert ofs_loc.is_reg() + regalloc.possibly_free_var(args[1]) + if args[3] is not args[4]: # more of the MESS described above + regalloc.possibly_free_var(args[3]) forbidden_vars = [args[4], srcaddr_box] dstaddr_box = TempBox() dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, forbidden_vars) self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, is_unicode=is_unicode) - regalloc.possibly_free_var(boxes[2]) - if args[3] is not args[4]: # more of the MESS described above - regalloc.possibly_free_var(boxes[3]) # compute the length in bytes length_loc, length_box = self._ensure_value_is_boxed(args[4], regalloc) - boxes.append(length_box) + args.append(length_box) if is_unicode: forbidden_vars = [srcaddr_box, dstaddr_box] bytes_box = TempBox() bytes_loc = regalloc.force_allocate_reg(bytes_box, forbidden_vars) scale = self._get_unicode_item_scale() + assert length_loc.is_reg() self.mc.MOV_rr(bytes_loc.value, length_loc.value) self._load_address(length_loc, 0, scale, bytes_loc) length_box = bytes_box @@ -739,8 +738,7 @@ regalloc.possibly_free_var(length_box) regalloc.possibly_free_var(dstaddr_box) regalloc.possibly_free_var(srcaddr_box) - regalloc.possibly_free_vars_for_op(op) - regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_vars(args) def _load_address(self, sizereg, baseofs, scale, result, baseloc=None): if baseloc is not None: Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Sat Dec 4 21:25:38 2010 @@ -1,5 +1,5 @@ from pypy.jit.backend.llsupport.regalloc import FrameManager, \ - RegisterManager, compute_vars_longevity + RegisterManager, compute_vars_longevity, TempBox from pypy.jit.backend.arm import registers as r from pypy.jit.backend.arm import locations from pypy.jit.metainterp.history import ConstInt, ConstPtr @@ -46,6 +46,24 @@ # is also used on op args, which is a non-resizable list self.possibly_free_vars(list(inputargs)) + def force_allocate_reg(self, v, forbidden_vars=[], selected_reg=None, + need_lower_byte=False): + # override ../llsupport/regalloc.py to set longevity for vals not in longevity + self._check_type(v) + if isinstance(v, TempBox) or v not in self.longevity: + self.longevity[v] = (self.position, self.position) + loc = self.try_allocate_reg(v, selected_reg, + need_lower_byte=need_lower_byte) + if loc: + return loc + loc = self._spill_var(v, forbidden_vars, selected_reg, + need_lower_byte=need_lower_byte) + prev_loc = self.reg_bindings.get(v, None) + if prev_loc is not None: + self.free_regs.append(prev_loc) + self.reg_bindings[v] = loc + return loc + class ARMFrameManager(FrameManager): def __init__(self): FrameManager.__init__(self) From afa at codespeak.net Sat Dec 4 22:02:03 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 4 Dec 2010 22:02:03 +0100 (CET) Subject: [pypy-svn] r79826 - in pypy/trunk/pypy/module/cpyext: . test Message-ID: <20101204210203.D6371282B9E@codespeak.net> Author: afa Date: Sat Dec 4 22:02:02 2010 New Revision: 79826 Modified: pypy/trunk/pypy/module/cpyext/pythonrun.py pypy/trunk/pypy/module/cpyext/state.py pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Log: Add Py_GetProgramName Modified: pypy/trunk/pypy/module/cpyext/pythonrun.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pythonrun.py (original) +++ pypy/trunk/pypy/module/cpyext/pythonrun.py Sat Dec 4 22:02:02 2010 @@ -1,6 +1,16 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.state import State @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def Py_IsInitialized(space): return 1 + + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) +def Py_GetProgramName(space): + """ + Return the program name set with Py_SetProgramName(), or the default. + The returned string points into static storage; the caller should not modify its + value.""" + return space.fromcache(State).get_programname() + Modified: pypy/trunk/pypy/module/cpyext/state.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/state.py (original) +++ pypy/trunk/pypy/module/cpyext/state.py Sat Dec 4 22:02:02 2010 @@ -8,6 +8,7 @@ def __init__(self, space): self.space = space self.reset() + self.programname = lltype.nullptr(rffi.CCHARP.TO) def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef @@ -41,3 +42,16 @@ if always: raise OperationError(self.space.w_SystemError, self.space.wrap( "Function returned an error result without setting an exception")) + + def get_programname(self): + if not self.programname: + space = self.space + argv = space.sys.get('argv') + if space.int_w(space.len(argv)): + argv0 = space.getitem(argv, space.wrap(0)) + progname = space.str_w(argv0) + else: + progname = "pypy" + self.programname = rffi.str2charp(progname) + lltype.render_immortal(self.programname) + return self.programname Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Sat Dec 4 22:02:02 2010 @@ -1422,17 +1422,6 @@ raise NotImplementedError @cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) -def Py_GetProgramName(space, ): - """ - - - - Return the program name set with Py_SetProgramName(), or the default. - The returned string points into static storage; the caller should not modify its - value.""" - raise NotImplementedError - - at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPrefix(space, ): """Return the prefix for installed platform-independent files. This is derived through a number of complicated rules from the program name set with Modified: pypy/trunk/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_cpyext.py Sat Dec 4 22:02:02 2010 @@ -682,3 +682,19 @@ assert mod.get_names() == ('cell', 'module', 'property', 'staticmethod', 'builtin_function_or_method') + + def test_get_programname(self): + mod = self.import_extension('foo', [ + ('get_programname', 'METH_NOARGS', + ''' + char* name1 = Py_GetProgramName(); + char* name2 = Py_GetProgramName(); + if (name1 != name2) + Py_RETURN_FALSE; + return PyString_FromString(name1); + ''' + ), + ]) + p = mod.get_programname() + print p + assert 'py' in p From benjamin at codespeak.net Sat Dec 4 22:16:39 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 4 Dec 2010 22:16:39 +0100 (CET) Subject: [pypy-svn] r79827 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20101204211639.7EA79282BAD@codespeak.net> Author: benjamin Date: Sat Dec 4 22:16:37 2010 New Revision: 79827 Modified: pypy/branch/fast-forward/pypy/objspace/std/frame.py Log: correct opcode Modified: pypy/branch/fast-forward/pypy/objspace/std/frame.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/frame.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/frame.py Sat Dec 4 22:16:37 2010 @@ -18,11 +18,11 @@ def LIST_APPEND(f, oparg, next_instr): w = f.popvalue() - v = f.peekvalue(oparg - 1) + v = f.peekvalue(oparg) if type(v) is W_ListObject: v.append(w) else: - f.space.call_method(v, 'append', w) + raise AssertionError def small_int_BINARY_ADD(f, oparg, next_instr): From benjamin at codespeak.net Sat Dec 4 22:49:19 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 4 Dec 2010 22:49:19 +0100 (CET) Subject: [pypy-svn] r79828 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20101204214919.B1056282BD4@codespeak.net> Author: benjamin Date: Sat Dec 4 22:49:18 2010 New Revision: 79828 Modified: pypy/branch/fast-forward/pypy/objspace/std/frame.py Log: revert Modified: pypy/branch/fast-forward/pypy/objspace/std/frame.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/frame.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/frame.py Sat Dec 4 22:49:18 2010 @@ -18,7 +18,7 @@ def LIST_APPEND(f, oparg, next_instr): w = f.popvalue() - v = f.peekvalue(oparg) + v = f.peekvalue(oparg - 1) if type(v) is W_ListObject: v.append(w) else: From afa at codespeak.net Sat Dec 4 23:15:34 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 4 Dec 2010 23:15:34 +0100 (CET) Subject: [pypy-svn] r79829 - pypy/trunk/pypy/module/cpyext/test Message-ID: <20101204221534.68CD1282B9C@codespeak.net> Author: afa Date: Sat Dec 4 23:15:32 2010 New Revision: 79829 Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Log: Clear exception state, and fix the test Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Sat Dec 4 23:15:32 2010 @@ -157,8 +157,7 @@ Py_RETURN_FALSE; PyErr_Restore(type, val, tb); - if (!PyErr_Occurred()) - Py_RETURN_FALSE; + PyErr_Clear(); Py_RETURN_TRUE; ''' ), From danchr at codespeak.net Sat Dec 4 23:44:22 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 23:44:22 +0100 (CET) Subject: [pypy-svn] r79830 - in pypy/trunk: . lib-python/modified-2.5.2/distutils pypy pypy/jit/metainterp/optimizeopt pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/module/cpyext/test pypy/rlib pypy/rlib/test pypy/tool/release pypy/translator/platform Message-ID: <20101204224422.C1ADA282B9C@codespeak.net> Author: danchr Date: Sat Dec 4 23:44:20 2010 New Revision: 79830 Modified: pypy/trunk/ (props changed) pypy/trunk/lib-python/modified-2.5.2/distutils/unixccompiler.py pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/trunk/pypy/module/array/benchmark/Makefile (props changed) pypy/trunk/pypy/module/array/benchmark/intimg.c (props changed) pypy/trunk/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/trunk/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/trunk/pypy/module/array/benchmark/loop.c (props changed) pypy/trunk/pypy/module/array/benchmark/sum.c (props changed) pypy/trunk/pypy/module/array/benchmark/sumtst.c (props changed) pypy/trunk/pypy/module/array/benchmark/sumtst.py (props changed) pypy/trunk/pypy/module/array/test/test_array_old.py (props changed) pypy/trunk/pypy/module/cpyext/object.py pypy/trunk/pypy/module/cpyext/pyerrors.py pypy/trunk/pypy/module/cpyext/stubs.py pypy/trunk/pypy/module/cpyext/test/test_object.py pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py pypy/trunk/pypy/rlib/rerased.py (props changed) pypy/trunk/pypy/rlib/test/test_rerased.py (props changed) pypy/trunk/pypy/tool/release/package.py pypy/trunk/pypy/translator/platform/posix.py Log: Merge branch cpyext-darwin This contains fixes to allow building C extensions on Darwin, and a few extra C APIs as well. Modified: pypy/trunk/lib-python/modified-2.5.2/distutils/unixccompiler.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/distutils/unixccompiler.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/distutils/unixccompiler.py Sat Dec 4 23:44:20 2010 @@ -121,7 +121,22 @@ } if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() executables['ranlib'] = ["ranlib"] + executables['linker_so'] += ['-undefined', 'dynamic_lookup'] + + for k, v in executables.iteritems(): + if v and v[0] == 'cc': + v += ['-arch', arch] + # Needed for the filename generation methods provided by the base # class, CCompiler. NB. whoever instantiates/uses a particular Modified: pypy/trunk/pypy/module/cpyext/object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/object.py (original) +++ pypy/trunk/pypy/module/cpyext/object.py Sat Dec 4 23:44:20 2010 @@ -2,12 +2,13 @@ from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, - Py_GE, CONST_STRING, FILEP, fwrite) + Py_GE, CONST_STRING, FILEP, fwrite, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference, get_typedescr, RefcountState) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall +from pypy.module._file.interp_file import W_File from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject from pypy.interpreter.error import OperationError @@ -428,6 +429,34 @@ rffi.free_nonmovingbuffer(data, buf) return 0 +PyFile_Check, PyFile_CheckExact = build_type_checkers("File", W_File) + + at cpython_api([PyObject, rffi.INT_real], PyObject) +def PyFile_GetLine(space, w_obj, n): + """ + Equivalent to p.readline([n]), this function reads one line from the + object p. p may be a file object or any object with a readline() + method. If n is 0, exactly one line is read, regardless of the length of + the line. If n is greater than 0, no more than n bytes will be read + from the file; a partial line can be returned. In both cases, an empty string + is returned if the end of the file is reached immediately. If n is less than + 0, however, one line is read regardless of length, but EOFError is + raised if the end of the file is reached immediately.""" + try: + w_readline = space.getattr(w_obj, space.wrap('readline')) + except OperationError: + raise OperationError( + space.w_TypeError, space.wrap( + "argument must be a file, or have a readline() method.")) + + n = rffi.cast(lltype.Signed, n) + if space.is_true(space.gt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline, space.wrap(n)) + elif space.is_true(space.lt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline) + else: + # XXX Raise EOFError as specified + return space.call_function(w_readline) @cpython_api([CONST_STRING, CONST_STRING], PyObject) def PyFile_FromString(space, filename, mode): """ @@ -437,4 +466,3 @@ w_filename = space.wrap(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) - Modified: pypy/trunk/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/pyerrors.py Sat Dec 4 23:44:20 2010 @@ -129,10 +129,29 @@ function around a system call can write return PyErr_SetFromErrno(type); when the system call returns an error. Return value: always NULL.""" + PyErr_SetFromErrnoWithFilename(space, w_type, + lltype.nullptr(rffi.CCHARP.TO)) + + at cpython_api([PyObject, rffi.CCHARP], PyObject) +def PyErr_SetFromErrnoWithFilename(space, w_type, llfilename): + """Similar to PyErr_SetFromErrno(), with the additional behavior that if + filename is not NULL, it is passed to the constructor of type as a third + parameter. In the case of exceptions such as IOError and OSError, + this is used to define the filename attribute of the exception instance. + Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() msg = os.strerror(errno) - w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + if llfilename: + w_filename = rffi.charp2str(llfilename) + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg), + space.wrap(w_filename)) + else: + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg)) raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) Modified: pypy/trunk/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/stubs.py (original) +++ pypy/trunk/pypy/module/cpyext/stubs.py Sat Dec 4 23:44:20 2010 @@ -668,15 +668,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.CCHARP], PyObject) -def PyErr_SetFromErrnoWithFilename(space, type, filename): - """Similar to PyErr_SetFromErrno(), with the additional behavior that if - filename is not NULL, it is passed to the constructor of type as a third - parameter. In the case of exceptions such as IOError and OSError, - this is used to define the filename attribute of the exception instance. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([rffi.INT_real], PyObject) def PyErr_SetFromWindowsErr(space, ierr): """This is a convenience function to raise WindowsError. If called with @@ -800,21 +791,6 @@ successful invocation of Py_EnterRecursiveCall().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_Check(space, p): - """Return true if its argument is a PyFileObject or a subtype of - PyFileObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_CheckExact(space, p): - """Return true if its argument is a PyFileObject, but not a subtype of - PyFileObject. - """ - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file @@ -848,22 +824,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], PyObject) -def PyFile_GetLine(space, p, n): - """ - - - - Equivalent to p.readline([n]), this function reads one line from the - object p. p may be a file object or any object with a readline() - method. If n is 0, exactly one line is read, regardless of the length of - the line. If n is greater than 0, no more than n bytes will be read - from the file; a partial line can be returned. In both cases, an empty string - is returned if the end of the file is reached immediately. If n is less than - 0, however, one line is read regardless of length, but EOFError is - raised if the end of the file is reached immediately.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyFile_Name(space, p): """Return the name of the file specified by p as a string object.""" Modified: pypy/trunk/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_object.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_object.py Sat Dec 4 23:44:20 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ - Py_GE, Py_GT + Py_GE, Py_GT, fopen, fclose, fwrite from pypy.tool.udir import udir class TestObject(BaseApiTest): @@ -188,10 +188,45 @@ rffi.free_charp(filename) rffi.free_charp(mode) + assert api.PyFile_Check(w_file) + assert api.PyFile_CheckExact(w_file) + assert not api.PyFile_Check(space.wrap("text")) + space.call_method(w_file, "write", space.wrap("text")) space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" + def test_file_getline(self, space, api): + filename = rffi.str2charp(str(udir / "_test_file")) + + mode = rffi.str2charp("w") + w_file = api.PyFile_FromString(filename, mode) + space.call_method(w_file, "write", + space.wrap("line1\nline2\nline3\nline4")) + space.call_method(w_file, "close") + + rffi.free_charp(mode) + mode = rffi.str2charp("r") + w_file = api.PyFile_FromString(filename, mode) + rffi.free_charp(filename) + rffi.free_charp(mode) + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "line1\n" + + w_line = api.PyFile_GetLine(w_file, 4) + assert space.str_w(w_line) == "line" + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "2\n" + + # XXX We ought to raise an EOFError here, but don't + w_line = api.PyFile_GetLine(w_file, -1) + # assert api.PyErr_Occurred() is space.w_EOFError + assert space.str_w(w_line) == "line3\n" + + space.call_method(w_file, "close") + class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) Modified: pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_pyerrors.py Sat Dec 4 23:44:20 2010 @@ -184,3 +184,26 @@ except OSError, e: assert e.errno == errno.EBADF assert e.strerror == os.strerror(errno.EBADF) + assert e.filename == None + + def test_SetFromErrnoWithFilename(self): + import sys + if sys.platform != 'win32': + skip("callbacks through ll2ctypes modify errno") + import errno, os + + module = self.import_extension('foo', [ + ("set_from_errno", "METH_NOARGS", + ''' + errno = EBADF; + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "blyf"); + return NULL; + '''), + ], + prologue="#include ") + try: + module.set_from_errno() + except OSError, e: + assert e.filename == "blyf" + assert e.errno == errno.EBADF + assert e.strerror == os.strerror(errno.EBADF) Modified: pypy/trunk/pypy/tool/release/package.py ============================================================================== --- pypy/trunk/pypy/tool/release/package.py (original) +++ pypy/trunk/pypy/tool/release/package.py Sat Dec 4 23:44:20 2010 @@ -78,7 +78,7 @@ old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) # ignore errors + os.system("strip -x " + str(archive_pypy_c)) # ignore errors if USE_TARFILE_MODULE: import tarfile tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') Modified: pypy/trunk/pypy/translator/platform/posix.py ============================================================================== --- pypy/trunk/pypy/translator/platform/posix.py (original) +++ pypy/trunk/pypy/translator/platform/posix.py Sat Dec 4 23:44:20 2010 @@ -166,7 +166,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'], - '$(CC_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') + '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') return m From danchr at codespeak.net Sat Dec 4 23:49:55 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sat, 4 Dec 2010 23:49:55 +0100 (CET) Subject: [pypy-svn] r79831 - pypy/branch/cpyext-darwin Message-ID: <20101204224955.D3B19282B9C@codespeak.net> Author: danchr Date: Sat Dec 4 23:49:54 2010 New Revision: 79831 Removed: pypy/branch/cpyext-darwin/ Log: Delete merged branch. From dan at codespeak.net Sun Dec 5 01:18:41 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Sun, 5 Dec 2010 01:18:41 +0100 (CET) Subject: [pypy-svn] r79832 - pypy/branch/psycopg2compatibility/pypy/module/cpyext/test Message-ID: <20101205001841.CDAC7282B9C@codespeak.net> Author: dan Date: Sun Dec 5 01:18:39 2010 New Revision: 79832 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_object.py Log: Not sure this is an adequate test, but it does fail with the bad code, and succeed with the good... Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_object.py Sun Dec 5 01:18:39 2010 @@ -232,3 +232,34 @@ """)]) assert module.dump(self.tmpname, None) assert open(self.tmpname).read() == 'None' + + def test_functioncallobjargs(self): + module = self.import_extension('callfunctiontest', [ + ("test_CallFunctionObjArgs", "METH_NOARGS", + """ + PyObject * value = PyString_FromString("13"); + PyObject * radix = PyInt_FromLong(16); + PyObject * result = PyObject_CallFunctionObjArgs((PyObject *)&PyInt_Type, value, radix, NULL); + + if (!PyInt_Check(result)) + { + Py_DECREF(value); + Py_DECREF(radix); + Py_DECREF(result); + Py_RETURN_FALSE; + } + + if (PyInt_AsLong(result) != 0x13) + { + Py_DECREF(value); + Py_DECREF(radix); + Py_DECREF(result); + Py_RETURN_FALSE; + } + + Py_DECREF(value); + Py_DECREF(radix); + Py_DECREF(result); + Py_RETURN_TRUE; + """)]) + assert module.test_CallFunctionObjArgs() From dan at codespeak.net Sun Dec 5 01:19:42 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Sun, 5 Dec 2010 01:19:42 +0100 (CET) Subject: [pypy-svn] r79833 - pypy/branch/psycopg2compatibility/pypy/module/cpyext Message-ID: <20101205001942.C7C75282B9E@codespeak.net> Author: dan Date: Sun Dec 5 01:19:41 2010 New Revision: 79833 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py Log: DEBUG_REFCOUNT works again! Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py Sun Dec 5 01:19:41 2010 @@ -235,8 +235,7 @@ ref = self.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) if not ref: if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone:", \ - hex(containee) + print >>sys.stderr, "Borrowed object is already gone:", w_obj return containee_ptr = rffi.cast(ADDR, ref) From fijal at codespeak.net Sun Dec 5 09:30:53 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 5 Dec 2010 09:30:53 +0100 (CET) Subject: [pypy-svn] r79834 - pypy/branch/remove-sys-recursionlimit/pypy/translator/goal Message-ID: <20101205083053.46F1B282B9C@codespeak.net> Author: fijal Date: Sun Dec 5 09:30:50 2010 New Revision: 79834 Modified: pypy/branch/remove-sys-recursionlimit/pypy/translator/goal/app_main.py Log: don't call sys.setrecursionlimit on pypy Modified: pypy/branch/remove-sys-recursionlimit/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/remove-sys-recursionlimit/pypy/translator/goal/app_main.py (original) +++ pypy/branch/remove-sys-recursionlimit/pypy/translator/goal/app_main.py Sun Dec 5 09:30:50 2010 @@ -312,8 +312,9 @@ cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 - # but we need more in the translated PyPy for the compiler package - sys.setrecursionlimit(5000) + # but we need more in the translated PyPy for the compiler package + if '__pypy__' not in sys.builtin_module_names: + sys.setrecursionlimit(5000) if unbuffered: set_unbuffered_io() From fijal at codespeak.net Sun Dec 5 09:36:47 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 5 Dec 2010 09:36:47 +0100 (CET) Subject: [pypy-svn] r79835 - in pypy/trunk: . pypy pypy/interpreter pypy/jit/metainterp/optimizeopt pypy/module/_stackless pypy/module/array/benchmark pypy/module/array/test pypy/module/sys pypy/rlib pypy/rlib/test pypy/translator/goal Message-ID: <20101205083647.87465282BD4@codespeak.net> Author: fijal Date: Sun Dec 5 09:36:45 2010 New Revision: 79835 Modified: pypy/trunk/ (props changed) pypy/trunk/pypy/ (props changed) pypy/trunk/pypy/interpreter/executioncontext.py pypy/trunk/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/trunk/pypy/module/_stackless/interp_coroutine.py pypy/trunk/pypy/module/array/benchmark/Makefile (props changed) pypy/trunk/pypy/module/array/benchmark/intimg.c (props changed) pypy/trunk/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/trunk/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/trunk/pypy/module/array/benchmark/loop.c (props changed) pypy/trunk/pypy/module/array/benchmark/sum.c (props changed) pypy/trunk/pypy/module/array/benchmark/sumtst.c (props changed) pypy/trunk/pypy/module/array/benchmark/sumtst.py (props changed) pypy/trunk/pypy/module/array/test/test_array_old.py (props changed) pypy/trunk/pypy/module/sys/vm.py pypy/trunk/pypy/rlib/rerased.py (props changed) pypy/trunk/pypy/rlib/test/test_rerased.py (props changed) pypy/trunk/pypy/translator/goal/app_main.py Log: Remove sys.setrecursionlimit functionality. It still works and you can get/set it, but it won't cause RuntimeError to be raised. The reason behind that is that we have better mechanism in place already that prevents this from happening (and does not have impact on JIT). Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Sun Dec 5 09:36:45 2010 @@ -27,7 +27,6 @@ def __init__(self, space): self.space = space self.topframeref = jit.vref_None - self.framestackdepth = 0 # tracing: space.frame_trace_action.fire() must be called to ensure # that tracing occurs whenever self.w_tracefunc or self.is_tracing # is modified. @@ -54,9 +53,6 @@ return frame def enter(self, frame): - if self.framestackdepth > self.space.sys.recursionlimit: - raise self.space.prebuilt_recursion_error - self.framestackdepth += 1 frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) @@ -66,7 +62,6 @@ self._trace(frame, 'leaveframe', self.space.w_None) finally: self.topframeref = frame.f_backref - self.framestackdepth -= 1 jit.virtual_ref_finish(frame) if self.w_tracefunc is not None and not frame.hide(): @@ -80,7 +75,6 @@ def __init__(self): self.topframe = None - self.framestackdepth = 0 self.w_tracefunc = None self.profilefunc = None self.w_profilefuncarg = None @@ -88,7 +82,6 @@ def enter(self, ec): ec.topframeref = jit.non_virtual_ref(self.topframe) - ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc ec.w_profilefuncarg = self.w_profilefuncarg @@ -97,7 +90,6 @@ def leave(self, ec): self.topframe = ec.gettopframe() - self.framestackdepth = ec.framestackdepth self.w_tracefunc = ec.w_tracefunc self.profilefunc = ec.profilefunc self.w_profilefuncarg = ec.w_profilefuncarg @@ -105,7 +97,6 @@ def clear_framestack(self): self.topframe = None - self.framestackdepth = 0 # the following interface is for pickling and unpickling def getstate(self, space): @@ -121,17 +112,14 @@ self.topframe = space.interp_w(PyFrame, frames_w[-1]) else: self.topframe = None - self.framestackdepth = len(frames_w) def getframestack(self): - index = self.framestackdepth - lst = [None] * index + lst = [] f = self.topframe - while index > 0: - index -= 1 - lst[index] = f + while f is not None: + lst.append(f) f = f.f_backref() - assert f is None + lst.reverse() return lst # coroutine: I think this is all, folks! Modified: pypy/trunk/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/trunk/pypy/module/_stackless/interp_coroutine.py Sun Dec 5 09:36:45 2010 @@ -304,16 +304,14 @@ def w_descr__framestack(space, self): assert isinstance(self, AppCoroutine) - index = self.subctx.framestackdepth - if not index: - return space.newtuple([]) - items = [None] * index f = self.subctx.topframe - while index > 0: - index -= 1 - items[index] = space.wrap(f) + items = [] + if not f: + return space.newtuple([]) + while f is not None: + items.append(space.wrap(f)) f = f.f_backref() - assert f is None + items.reverse() return space.newtuple(items) def makeStaticMethod(module, classname, funcname): Modified: pypy/trunk/pypy/module/sys/vm.py ============================================================================== --- pypy/trunk/pypy/module/sys/vm.py (original) +++ pypy/trunk/pypy/module/sys/vm.py Sun Dec 5 09:36:45 2010 @@ -41,27 +41,21 @@ f = ec.getnextframe_nohidden(f) return space.wrap(f) -# directly from the C code in ceval.c, might be moved somewhere else. - def setrecursionlimit(space, w_new_limit): - """Set the maximum depth of the Python interpreter stack to n. This -limit prevents infinite recursion from causing an overflow of the C -stack and crashing Python. The highest possible limit is platform -dependent.""" + """DEPRECATED on PyPy. Will issue warning and not work + """ new_limit = space.int_w(w_new_limit) if new_limit <= 0: raise OperationError(space.w_ValueError, space.wrap("recursion limit must be positive")) # global recursion_limit # we need to do it without writing globals. + space.warn('setrecursionlimit deprecated', space.w_DeprecationWarning) space.sys.recursionlimit = new_limit def getrecursionlimit(space): - """Return the current value of the recursion limit, the maximum depth - of the Python interpreter stack. This limit prevents infinite - recursion from causing an overflow of the C stack and crashing Python. + """DEPRECATED on PyPy. Will issue warning and not work """ - return space.wrap(space.sys.recursionlimit) def setcheckinterval(space, interval): Modified: pypy/trunk/pypy/translator/goal/app_main.py ============================================================================== --- pypy/trunk/pypy/translator/goal/app_main.py (original) +++ pypy/trunk/pypy/translator/goal/app_main.py Sun Dec 5 09:36:45 2010 @@ -312,8 +312,9 @@ cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 - # but we need more in the translated PyPy for the compiler package - sys.setrecursionlimit(5000) + # but we need more in the translated PyPy for the compiler package + if '__pypy__' not in sys.builtin_module_names: + sys.setrecursionlimit(5000) if unbuffered: set_unbuffered_io() From danchr at codespeak.net Sun Dec 5 15:30:01 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sun, 5 Dec 2010 15:30:01 +0100 (CET) Subject: [pypy-svn] r79836 - pypy/trunk/pypy/translator/platform Message-ID: <20101205143001.4E81C282BAD@codespeak.net> Author: danchr Date: Sun Dec 5 15:29:59 2010 New Revision: 79836 Modified: pypy/trunk/pypy/translator/platform/darwin.py Log: Default to GCC 4.0 on Darwin/i386 and not only Darwin/x86_64 The default compiler on Mac OS X 10.6 is Apple GCC 4.2. Using it causes problems with GC root detection. Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Sun Dec 5 15:29:59 2010 @@ -12,7 +12,7 @@ so_ext = 'so' - default_cc = 'gcc' + default_cc = 'gcc-4.0' def __init__(self, cc=None): if cc is None: @@ -87,4 +87,3 @@ link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4') cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4') - default_cc = 'gcc-4.0' From danchr at codespeak.net Sun Dec 5 17:08:19 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Sun, 5 Dec 2010 17:08:19 +0100 (CET) Subject: [pypy-svn] r79837 - pypy/trunk/pypy/translator/platform Message-ID: <20101205160819.74A0A282BAD@codespeak.net> Author: danchr Date: Sun Dec 5 17:08:16 2010 New Revision: 79837 Modified: pypy/trunk/pypy/translator/platform/darwin.py Log: Add a note explaining why GCC 4.0 is used on Darwin. Modified: pypy/trunk/pypy/translator/platform/darwin.py ============================================================================== --- pypy/trunk/pypy/translator/platform/darwin.py (original) +++ pypy/trunk/pypy/translator/platform/darwin.py Sun Dec 5 17:08:16 2010 @@ -11,7 +11,9 @@ shared_only = () so_ext = 'so' - + + # NOTE: GCC 4.2 will fail at runtime due to subtle issues, possibly + # related to GC roots. Using LLVM-GCC or Clang will break the build. default_cc = 'gcc-4.0' def __init__(self, cc=None): From hakanardo at codespeak.net Sun Dec 5 18:04:39 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sun, 5 Dec 2010 18:04:39 +0100 (CET) Subject: [pypy-svn] r79838 - in pypy/branch/jit-unroll-loops/pypy/jit/metainterp: . test Message-ID: <20101205170439.89B05282BAD@codespeak.net> Author: hakanardo Date: Sun Dec 5 18:04:37 2010 New Revision: 79838 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmspot.py Log: Always generate the full preambles, they are used as targets of the guards inlined from the short preamble Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/pyjitpl.py Sun Dec 5 18:04:37 2010 @@ -1849,7 +1849,7 @@ self.history.inputargs = original_boxes[num_green_args:] greenkey = original_boxes[:num_green_args] self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) - loop_token = compile.compile_new_loop(self, [], greenkey, start, False) + loop_token = compile.compile_new_loop(self, [], greenkey, start) self.history.operations.pop() # remove the JUMP if loop_token is None: return Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py Sun Dec 5 18:04:37 2010 @@ -1840,6 +1840,46 @@ 'int_add': 1, 'int_mul': 1, 'int_sub': 2, 'int_gt': 2, 'jump': 2}) + def test_multiple_specialied_versions_bridge(self): + myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'z', 'res']) + class Base: + def __init__(self, val): + self.val = val + def getval(self): + return self.val + class A(Base): + def binop(self, other): + return A(self.getval() + other.getval()) + class B(Base): + def binop(self, other): + return B(self.getval() * other.getval()) + def f(x, y, z): + res = x + while y > 0: + myjitdriver.can_enter_jit(y=y, x=x, z=z, res=res) + myjitdriver.jit_merge_point(y=y, x=x, z=z, res=res) + res = res.binop(x) + y -= 1 + if y < 7: + x = z + return res + def g(x, y): + a1 = f(A(x), y, A(x)) + a2 = f(A(x), y, A(x)) + b1 = f(B(x), y, B(x)) + b2 = f(B(x), y, B(x)) + c1 = f(B(x), y, A(x)) + c2 = f(B(x), y, A(x)) + d1 = f(A(x), y, B(x)) + d2 = f(A(x), y, B(x)) + assert a1.val == a2.val + assert b1.val == b2.val + assert c1.val == c2.val + assert d1.val == d2.val + return a1.val + b1.val + c1.val + d1.val + res = self.meta_interp(g, [6, 14]) + assert res == g(6, 14) + def test_specialied_bridge(self): myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'res']) class A: Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmspot.py Sun Dec 5 18:04:37 2010 @@ -321,7 +321,7 @@ return 'DoneWithThisFrameVoid()' class DoneWithThisFrameInt(JitException): - def __init__(self, result): + def __init__(self, result): assert lltype.typeOf(result) is lltype.Signed self.result = result def __str__(self): From arigo at codespeak.net Mon Dec 6 11:01:56 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 11:01:56 +0100 (CET) Subject: [pypy-svn] r79839 - in pypy/trunk/pypy/jit/codewriter: . test Message-ID: <20101206100156.E6111282B90@codespeak.net> Author: arigo Date: Mon Dec 6 11:01:54 2010 New Revision: 79839 Modified: pypy/trunk/pypy/jit/codewriter/regalloc.py pypy/trunk/pypy/jit/codewriter/test/test_regalloc.py Log: Very minor test and fix: if an operation produces an unused result, it remains accidentally occupying a variable until the end of the block. Modified: pypy/trunk/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/regalloc.py (original) +++ pypy/trunk/pypy/jit/codewriter/regalloc.py Mon Dec 6 11:01:54 2010 @@ -36,7 +36,7 @@ if isinstance(v1, Variable): die_at[v1] = i if op.result is not None: - die_at[op.result] = i + die_at[op.result] = i + 1 if isinstance(block.exitswitch, tuple): for x in block.exitswitch: die_at.pop(x, None) Modified: pypy/trunk/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/trunk/pypy/jit/codewriter/test/test_regalloc.py Mon Dec 6 11:01:54 2010 @@ -281,22 +281,22 @@ # this used to produce bogus code, containing these two # lines in the following broken order: # last_exc_value -> %r0 - # ref_copy %r0 -> %r2 -- but expect to read the old value of %r0! + # ref_copy %r0 -> %r1 -- but expect to read the old value of %r0! self.check_assembler(graph, """ residual_call_r_r $<* fn bar>, , R[%r0] -> %r1 -live- - residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r2 + residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r1 -live- catch_exception L1 - ref_return %r2 + ref_return %r1 --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 - ref_copy %r0 -> %r2 + ref_copy %r0 -> %r1 last_exc_value -> %r0 residual_call_r_r $<* fn foo>, , R[%r0] -> %r0 -live- - ref_return %r2 + ref_return %r1 --- L2: reraise From arigo at codespeak.net Mon Dec 6 11:25:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 11:25:25 +0100 (CET) Subject: [pypy-svn] r79840 - pypy/trunk/pypy/translator/platform Message-ID: <20101206102525.4284C282B90@codespeak.net> Author: arigo Date: Mon Dec 6 11:25:23 2010 New Revision: 79840 Modified: pypy/trunk/pypy/translator/platform/posix.py Log: Revert this part of r79830. It breaks on Linux, probably because of passing "-shared" when building a stand-alone executable. Modified: pypy/trunk/pypy/translator/platform/posix.py ============================================================================== --- pypy/trunk/pypy/translator/platform/posix.py (original) +++ pypy/trunk/pypy/translator/platform/posix.py Mon Dec 6 11:25:23 2010 @@ -166,7 +166,7 @@ 'int main(int argc, char* argv[]) ' '{ return $(PYPY_MAIN_FUNCTION)(argc, argv); }" > $@') m.rule('$(DEFAULT_TARGET)', ['$(TARGET)', 'main.o'], - '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') + '$(CC_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@') return m From arigo at codespeak.net Mon Dec 6 11:30:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 11:30:24 +0100 (CET) Subject: [pypy-svn] r79841 - pypy/trunk/pypy/module/_stackless Message-ID: <20101206103024.4FD68282BDC@codespeak.net> Author: arigo Date: Mon Dec 6 11:30:22 2010 New Revision: 79841 Modified: pypy/trunk/pypy/module/_stackless/interp_coroutine.py Log: Fix for stackless translations. Modified: pypy/trunk/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/trunk/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/trunk/pypy/module/_stackless/interp_coroutine.py Mon Dec 6 11:30:22 2010 @@ -304,14 +304,19 @@ def w_descr__framestack(space, self): assert isinstance(self, AppCoroutine) + counter = 0 f = self.subctx.topframe - items = [] - if not f: - return space.newtuple([]) while f is not None: - items.append(space.wrap(f)) + counter += 1 f = f.f_backref() - items.reverse() + items = [None] * counter + f = self.subctx.topframe + while f is not None: + counter -= 1 + assert counter >= 0 + items[counter] = space.wrap(f) + f = f.f_backref() + assert counter == 0 return space.newtuple(items) def makeStaticMethod(module, classname, funcname): From arigo at codespeak.net Mon Dec 6 11:38:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 11:38:38 +0100 (CET) Subject: [pypy-svn] r79842 - in pypy/branch/inline-shadowstack/pypy: config jit/codewriter jit/codewriter/test tool/algo tool/algo/test translator/c translator/c/src Message-ID: <20101206103838.B85E2282BDD@codespeak.net> Author: arigo Date: Mon Dec 6 11:38:36 2010 New Revision: 79842 Added: pypy/branch/inline-shadowstack/pypy/tool/algo/regalloc.py - copied, changed from r79813, pypy/branch/inline-shadowstack/pypy/jit/codewriter/regalloc.py pypy/branch/inline-shadowstack/pypy/tool/algo/test/test_regalloc.py Removed: pypy/branch/inline-shadowstack/pypy/jit/codewriter/regalloc.py Modified: pypy/branch/inline-shadowstack/pypy/config/translationoption.py pypy/branch/inline-shadowstack/pypy/jit/codewriter/codewriter.py pypy/branch/inline-shadowstack/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/inline-shadowstack/pypy/translator/c/funcgen.py pypy/branch/inline-shadowstack/pypy/translator/c/gc.py pypy/branch/inline-shadowstack/pypy/translator/c/genc.py pypy/branch/inline-shadowstack/pypy/translator/c/src/mem.h Log: First implementation. The results are not great on targetgcbench. Modified: pypy/branch/inline-shadowstack/pypy/config/translationoption.py ============================================================================== --- pypy/branch/inline-shadowstack/pypy/config/translationoption.py (original) +++ pypy/branch/inline-shadowstack/pypy/config/translationoption.py Mon Dec 6 11:38:36 2010 @@ -86,14 +86,16 @@ default=IS_64_BITS, cmdline="--gcremovetypeptr"), ChoiceOption("gcrootfinder", "Strategy for finding GC Roots (framework GCs only)", - ["n/a", "shadowstack", "asmgcc"], + ["n/a", "shadowstack", "asmgcc", "inlinestack"], "shadowstack", cmdline="--gcrootfinder", requires={ - "shadowstack": [("translation.gctransformer", "framework")], - "asmgcc": [("translation.gctransformer", "framework"), - ("translation.backend", "c")], - }), + "shadowstack": [("translation.gctransformer", "framework")], + "asmgcc": [("translation.gctransformer", "framework"), + ("translation.backend", "c")], + "inlinestack": [("translation.gctransformer", "framework"), + ("translation.backend", "c")], + }), # other noticeable options BoolOption("thread", "enable use of threading primitives", Modified: pypy/branch/inline-shadowstack/pypy/jit/codewriter/codewriter.py ============================================================================== --- pypy/branch/inline-shadowstack/pypy/jit/codewriter/codewriter.py (original) +++ pypy/branch/inline-shadowstack/pypy/jit/codewriter/codewriter.py Mon Dec 6 11:38:36 2010 @@ -1,5 +1,6 @@ from pypy.jit.codewriter import support, heaptracker -from pypy.jit.codewriter.regalloc import perform_register_allocation +from pypy.tool.algo.regalloc import perform_register_allocation +from pypy.jit.metainterp.history import getkind from pypy.jit.codewriter.flatten import flatten_graph, KINDS from pypy.jit.codewriter.assembler import Assembler, JitCode from pypy.jit.codewriter.jtransform import transform_graph @@ -43,7 +44,8 @@ # step 2: perform register allocation on it regallocs = {} for kind in KINDS: - regallocs[kind] = perform_register_allocation(graph, kind) + kind_filter = lambda v: getkind(v.concretetype) == kind + regallocs[kind] = perform_register_allocation(graph, kind_filter) # # step 3: flatten the graph to produce human-readable "assembler", # which means mostly producing a linear list of operations and Modified: pypy/branch/inline-shadowstack/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/inline-shadowstack/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/inline-shadowstack/pypy/jit/codewriter/test/test_regalloc.py Mon Dec 6 11:38:36 2010 @@ -1,9 +1,9 @@ import py, sys from pypy.jit.codewriter import support -from pypy.jit.codewriter.regalloc import perform_register_allocation +from pypy.tool.algo.regalloc import perform_register_allocation from pypy.jit.codewriter.flatten import flatten_graph, ListOfKind from pypy.jit.codewriter.format import assert_format -from pypy.jit.metainterp.history import AbstractDescr +from pypy.jit.metainterp.history import AbstractDescr, getkind from pypy.objspace.flow.model import Variable, Constant, SpaceOperation from pypy.objspace.flow.model import FunctionGraph, Block, Link from pypy.objspace.flow.model import c_last_exception @@ -18,6 +18,10 @@ self.rtyper = support.annotate(func, values, type_system=type_system) return self.rtyper.annotator.translator.graphs + def perform_register_allocation(self, graph, kind): + kind_filter = lambda v: getkind(v.concretetype) == kind + return perform_register_allocation(graph, kind_filter) + def check_assembler(self, graph, expected, transform=False, callcontrol=None): # 'transform' can be False only for simple graphs. More complex @@ -26,8 +30,8 @@ if transform: from pypy.jit.codewriter.jtransform import transform_graph transform_graph(graph, callcontrol=callcontrol) - regalloc = perform_register_allocation(graph, 'int') - regalloc2 = perform_register_allocation(graph, 'ref') + regalloc = self.perform_register_allocation(graph, 'int') + regalloc2 = self.perform_register_allocation(graph, 'ref') ssarepr = flatten_graph(graph, {'int': regalloc, 'ref': regalloc2}) assert_format(ssarepr, expected) @@ -36,7 +40,7 @@ def f(a, b): return a + b graph = self.make_graphs(f, [5, 6])[0] - regalloc = perform_register_allocation(graph, 'int') + regalloc = self.perform_register_allocation(graph, 'int') va, vb = graph.startblock.inputargs vc = graph.startblock.operations[0].result assert regalloc.getcolor(va) == 0 @@ -50,7 +54,7 @@ a -= 1 return b graph = self.make_graphs(f, [5, 6])[0] - regalloc = perform_register_allocation(graph, 'float') + regalloc = self.perform_register_allocation(graph, 'float') # assert did not crash def test_regalloc_loop(self): @@ -285,18 +289,18 @@ self.check_assembler(graph, """ residual_call_r_r $<* fn bar>, , R[%r0] -> %r1 -live- - residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r2 + residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r1 -live- catch_exception L1 - ref_return %r2 + ref_return %r1 --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 - ref_copy %r0 -> %r2 + ref_copy %r0 -> %r1 last_exc_value -> %r0 residual_call_r_r $<* fn foo>, , R[%r0] -> %r0 -live- - ref_return %r2 + ref_return %r1 --- L2: reraise Copied: pypy/branch/inline-shadowstack/pypy/tool/algo/regalloc.py (from r79813, pypy/branch/inline-shadowstack/pypy/jit/codewriter/regalloc.py) ============================================================================== --- pypy/branch/inline-shadowstack/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/inline-shadowstack/pypy/tool/algo/regalloc.py Mon Dec 6 11:38:36 2010 @@ -2,13 +2,12 @@ from pypy.objspace.flow.model import Variable from pypy.tool.algo.color import DependencyGraph from pypy.tool.algo.unionfind import UnionFind -from pypy.jit.metainterp.history import getkind -from pypy.jit.codewriter.flatten import ListOfKind -def perform_register_allocation(graph, kind): - """Perform register allocation for the Variables of the given 'kind' - in the 'graph'.""" - regalloc = RegAllocator(graph, kind) +def perform_register_allocation(graph, kind_filter, identity_op_filter=None): + """Perform register allocation for the Variables of the given kind + in the graph. + """ + regalloc = RegAllocator(graph, kind_filter, identity_op_filter) regalloc.make_dependencies() regalloc.coalesce_variables() regalloc.find_node_coloring() @@ -18,12 +17,18 @@ class RegAllocator(object): DEBUG_REGALLOC = False - def __init__(self, graph, kind): + def __init__(self, graph, kind_filter, identity_op_filter=None): self.graph = graph - self.kind = kind + self.kind_filter = kind_filter + if identity_op_filter is not None: + self.identity_op_filter = identity_op_filter + + def identity_op_filter(self, op): + return False # default implementation def make_dependencies(self): dg = DependencyGraph() + uf = UnionFind() for block in self.graph.iterblocks(): # Compute die_at = {Variable: index_of_operation_with_last_usage} die_at = dict.fromkeys(block.inputargs, 0) @@ -31,13 +36,13 @@ for v in op.args: if isinstance(v, Variable): die_at[v] = i - elif isinstance(v, ListOfKind): + elif is_iterable(v): for v1 in v: if isinstance(v1, Variable): die_at[v1] = i if op.result is not None: - die_at[op.result] = i - if isinstance(block.exitswitch, tuple): + die_at[op.result] = i + 1 + if is_iterable(block.exitswitch): for x in block.exitswitch: die_at.pop(x, None) else: @@ -48,10 +53,10 @@ die_at = [(value, key) for (key, value) in die_at.items()] die_at.sort() die_at.append((sys.maxint,)) - # Done. XXX the code above this line runs 3 times - # (for kind in KINDS) to produce the same result... - livevars = [v for v in block.inputargs - if getkind(v.concretetype) == self.kind] + # Done. XXX if we need to perform register allocation on + # the same graph with various 'kinds', the code above this + # line runs several times to produce the same result... + livevars = [v for v in block.inputargs if self.kind_filter(v)] # Add the variables of this block to the dependency graph for i, v in enumerate(livevars): dg.add_node(v) @@ -66,17 +71,21 @@ except KeyError: pass die_index += 1 - if (op.result is not None and - getkind(op.result.concretetype) == self.kind): - dg.add_node(op.result) - for v in livevars: - if getkind(v.concretetype) == self.kind: - dg.add_edge(v, op.result) + if op.result is not None and self.kind_filter(op.result): + if self.identity_op_filter(op): + rep1 = uf.find_rep(op.args[0]) + _, rep2, _ = uf.union(rep1, op.result) + assert rep2 is rep1 + else: + dg.add_node(op.result) + for v in livevars: + assert self.kind_filter(v) + dg.add_edge(uf.find_rep(v), op.result) livevars.add(op.result) self._depgraph = dg + self._unionfind = uf def coalesce_variables(self): - self._unionfind = UnionFind() pendingblocks = list(self.graph.iterblocks()) while pendingblocks: block = pendingblocks.pop() @@ -95,7 +104,7 @@ self._try_coalesce(v, link.target.inputargs[i]) def _try_coalesce(self, v, w): - if isinstance(v, Variable) and getkind(v.concretetype) == self.kind: + if isinstance(v, Variable) and self.kind_filter(v): dg = self._depgraph uf = self._unionfind v0 = uf.find_rep(v) @@ -126,3 +135,11 @@ self._coloring[key] = col2 elif value == col2: self._coloring[key] = col1 + + +def is_iterable(x): + try: + iter(x) + return True + except (TypeError, AttributeError): + return False Added: pypy/branch/inline-shadowstack/pypy/tool/algo/test/test_regalloc.py ============================================================================== --- (empty file) +++ pypy/branch/inline-shadowstack/pypy/tool/algo/test/test_regalloc.py Mon Dec 6 11:38:36 2010 @@ -0,0 +1,82 @@ + +# XXX not enough tests, but see also jit/codewriter/test/test_regalloc.py + +import py +from pypy.tool.algo.regalloc import perform_register_allocation +from pypy.rpython.test.test_llinterp import gengraph +from pypy.rpython.lltypesystem import lltype +from pypy.rlib.jit import hint + + +def test_simple(): + def f(a, b, c): + return (b * c) + a + def kind_float(v): + return v.concretetype == lltype.Float + + t, rtyper, graph = gengraph(f, [int, int, float]) + assert [op.opname for op in graph.startblock.operations] == [ + "cast_int_to_float", # (1) = b + "float_mul", # (0) = (1) * (0) + "cast_int_to_float", # (1) = a + "float_add"] # (0) = (1) + (0) + + regalloc = perform_register_allocation(graph, kind_float) + + py.test.raises(KeyError, regalloc.getcolor, graph.getargs()[0]) + py.test.raises(KeyError, regalloc.getcolor, graph.getargs()[1]) + + ops = graph.startblock.operations + assert regalloc.getcolor(graph.getargs()[2]) == 0 + assert regalloc.getcolor(ops[0].result) == 1 + assert regalloc.getcolor(ops[1].result) == 0 + assert regalloc.getcolor(ops[2].result) == 1 + assert regalloc.getcolor(ops[3].result) == 0 + assert regalloc.getcolor(graph.getreturnvar()) == 0 + +def test_unused_result(): + def f(x): + hint(x, blah=True) + hint(x, blah=True) + hint(x, blah=True) + return x + def kind_float(v): + return v.concretetype == lltype.Float + + t, rtyper, graph = gengraph(f, [lltype.Float]) + assert [op.opname for op in graph.startblock.operations] == [ + "hint", # (1) = hint(0) + "hint", # (1) = hint(0) + "hint"] # (1) = hint(0) + + regalloc = perform_register_allocation(graph, kind_float) + ops = graph.startblock.operations + assert regalloc.getcolor(graph.getargs()[0]) == 0 + assert regalloc.getcolor(ops[0].result) == 1 + assert regalloc.getcolor(ops[1].result) == 1 + assert regalloc.getcolor(ops[2].result) == 1 + assert regalloc.getcolor(graph.getreturnvar()) == 0 + +def test_identity_op(): + def f(x): + y = hint(x, blah=True) + z = hint(y, blah=True) + t = hint(x, blah=True) + return 0 + def kind_float(v): + return v.concretetype == lltype.Float + def identity_op(op): + return op.opname == 'hint' + + t, rtyper, graph = gengraph(f, [lltype.Float]) + assert [op.opname for op in graph.startblock.operations] == [ + "hint", # (0) = hint(0) + "hint", # (0) = hint(0) + "hint"] # (0) = hint(0) + + regalloc = perform_register_allocation(graph, kind_float, identity_op) + ops = graph.startblock.operations + assert regalloc.getcolor(graph.getargs()[0]) == 0 + assert regalloc.getcolor(ops[0].result) == 0 + assert regalloc.getcolor(ops[1].result) == 0 + assert regalloc.getcolor(ops[2].result) == 0 Modified: pypy/branch/inline-shadowstack/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/inline-shadowstack/pypy/translator/c/funcgen.py (original) +++ pypy/branch/inline-shadowstack/pypy/translator/c/funcgen.py Mon Dec 6 11:38:36 2010 @@ -176,6 +176,9 @@ if self.lltypemap(v) is Void and special_case_void: return '/* nothing */' else: + s = self.gcpolicy.special_ref_name_for_local(v) + if s: + return s return LOCALVAR % v.name elif isinstance(v, Constant): value = llvalue_from_constant(v) @@ -190,6 +193,7 @@ def cfunction_declarations(self): # declare the local variables, excluding the function arguments + self.gcpolicy.prepare_declarations_in_function(self) seen = {} for a in self.graph.getargs(): seen[a.name] = True @@ -199,6 +203,8 @@ name = v.name if name not in seen: seen[name] = True + if self.gcpolicy.special_ref_name_for_local(v): + continue result = cdecl(self.lltypename(v), LOCALVAR % name) + ';' if self.lltypemap(v) is Void: continue #result = '/*%s*/' % result @@ -209,6 +215,8 @@ # ____________________________________________________________ def cfunction_body(self): + for line in self.gcpolicy.extra_declarations_in_function(self): + yield line graph = self.graph yield 'goto block0;' # to avoid a warning "this label is not used" @@ -232,6 +240,8 @@ retval = self.expr(block.inputargs[0]) if self.exception_policy != "exc_helper": yield 'RPY_DEBUG_RETURN();' + for line in self.gcpolicy.extra_code_at_return(self): + yield line yield 'return %s;' % retval continue elif block.exitswitch is None: @@ -306,7 +316,7 @@ if a2type is Void: continue src = self.expr(a1) - dest = LOCALVAR % a2.name + dest = self.expr(a2) assignments.append((a2typename, dest, src)) for line in gen_assignments(assignments): yield line Modified: pypy/branch/inline-shadowstack/pypy/translator/c/gc.py ============================================================================== --- pypy/branch/inline-shadowstack/pypy/translator/c/gc.py (original) +++ pypy/branch/inline-shadowstack/pypy/translator/c/gc.py Mon Dec 6 11:38:36 2010 @@ -1,12 +1,12 @@ import sys -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Constant, Variable from pypy.translator.c.support import cdecl from pypy.translator.c.node import ContainerNode from pypy.rpython.lltypesystem.lltype import \ typeOf, Ptr, ContainerType, RttiStruct, \ RuntimeTypeInfo, getRuntimeTypeInfo, top_container from pypy.rpython.memory.gctransform import \ - refcounting, boehm, framework, asmgcroot + refcounting, boehm, framework, asmgcroot, inlinestack from pypy.rpython.lltypesystem import lltype, llmemory from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -70,6 +70,15 @@ def rtti_type(self): return '' + def prepare_declarations_in_function(self, funcgen): + pass + def special_ref_name_for_local(self, v): + return False + def extra_declarations_in_function(self, funcgen): + return [] + def extra_code_at_return(self, funcgen): + return [] + def OP_GC_PUSH_ALIVE_PYOBJ(self, funcgen, op): expr = funcgen.expr(op.args[0]) if expr == 'NULL': @@ -384,6 +393,65 @@ def OP_GC_STACK_BOTTOM(self, funcgen, op): return 'pypy_asm_stack_bottom();' +class InlineStackFrameworkGcPolicy(FrameworkGcPolicy): + transformerclass = inlinestack.InlineStackFrameworkGCTransformer + + def prepare_declarations_in_function(self, funcgen): + self.gcvar2index = {} # xxx should be stored on 'funcgen', ideally + self.all_gcvars = [] + for block in funcgen.graph.iterblocks(): + for op in block.operations: + if op.opname == 'gc_push_roots': + for v in op.args: + if not isinstance(v, Variable): + continue + assert funcgen.lltypemap(v) is not lltype.Void + vname = v.name + if vname not in self.gcvar2index: + self.gcvar2index[vname] = len(self.all_gcvars) + self.all_gcvars.append((funcgen.lltypename(v), + vname)) + + def special_ref_name_for_local(self, v): + if v.name in self.gcvar2index: + return 'ref.%s' % v.name + return None + + def extra_declarations_in_function(self, funcgen): + if self.all_gcvars: + yield 'struct {' + yield '\tstruct pypy_stackref_s hdr;' + assert len(self.all_gcvars) < 32 # XXX fix this limitation + for vtype, vname in self.all_gcvars: + yield '\t%s;' % cdecl(vtype, vname) + yield '} ref;' + # + yield 'ref.hdr.prev = pypy_stackref;' + yield 'pypy_stackref = &ref.hdr;' + for argname, v in zip(funcgen.argnames(), funcgen.graph.getargs()): + s = self.special_ref_name_for_local(v) + if s: + yield '%s = %s;' % (s, argname) + + def extra_code_at_return(self, funcgen): + if self.all_gcvars: + yield 'pypy_stackref = ref.hdr.prev;' + + def OP_GC_PUSH_ROOTS(self, funcgen, op): + bitfield = 0 + vars = [] + for v in op.args: + if isinstance(v, Variable): + bitfield |= 1 << self.gcvar2index[v.name] + vars.append(v.name) + if not self.all_gcvars: + assert not vars + return '' + result = 'ref.hdr.bitfield = %d;' % bitfield + if vars: + result += ' /* %s */' % (' '.join(vars),) + return result + name_to_gcpolicy = { 'boehm': BoehmGcPolicy, @@ -391,6 +459,5 @@ 'none': NoneGcPolicy, 'framework': FrameworkGcPolicy, 'framework+asmgcroot': AsmGcRootFrameworkGcPolicy, + 'framework+inlinestack': InlineStackFrameworkGcPolicy, } - - Modified: pypy/branch/inline-shadowstack/pypy/translator/c/genc.py ============================================================================== --- pypy/branch/inline-shadowstack/pypy/translator/c/genc.py (original) +++ pypy/branch/inline-shadowstack/pypy/translator/c/genc.py Mon Dec 6 11:38:36 2010 @@ -205,6 +205,8 @@ name = self.config.translation.gctransformer if self.config.translation.gcrootfinder == "asmgcc": name = "%s+asmgcroot" % (name,) + elif self.config.translation.gcrootfinder == "inlinestack": + name = "%s+inlinestack" % (name,) return gc.name_to_gcpolicy[name] return self.gcpolicy Modified: pypy/branch/inline-shadowstack/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/inline-shadowstack/pypy/translator/c/src/mem.h (original) +++ pypy/branch/inline-shadowstack/pypy/translator/c/src/mem.h Mon Dec 6 11:38:36 2010 @@ -9,6 +9,18 @@ extern long pypy_asm_stackwalk(void*, void*); #define __gcnoreorderhack __gcmapend +struct pypy_stackref_s { + struct pypy_stackref_s *prev; + long bitfield; +}; +extern struct pypy_stackref_s *pypy_stackref; +extern void *get_pypy_stackref(void); + +#ifndef PYPY_NOT_MAIN_FILE +struct pypy_stackref_s *pypy_stackref = NULL; +void *get_pypy_stackref(void) { return pypy_stackref; } +#endif + /* The following pseudo-instruction is used by --gcrootfinder=asmgcc just after a call to tell gcc to put a GCROOT mark on each gc-pointer local variable. All such local variables need to go through a "v = From arigo at codespeak.net Mon Dec 6 11:46:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 11:46:54 +0100 (CET) Subject: [pypy-svn] r79843 - pypy/branch/inline-shadowstack/pypy/rpython/memory/gctransform Message-ID: <20101206104654.118E2282BE9@codespeak.net> Author: arigo Date: Mon Dec 6 11:46:52 2010 New Revision: 79843 Added: pypy/branch/inline-shadowstack/pypy/rpython/memory/gctransform/inlinestack.py Log: Forgot to add. Added: pypy/branch/inline-shadowstack/pypy/rpython/memory/gctransform/inlinestack.py ============================================================================== --- (empty file) +++ pypy/branch/inline-shadowstack/pypy/rpython/memory/gctransform/inlinestack.py Mon Dec 6 11:46:52 2010 @@ -0,0 +1,71 @@ +from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer +from pypy.rpython.memory.gctransform.framework import BaseRootWalker +from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython import annlowlevel + +# +# This transformer works by letting the C backend handles most of the actual +# code, which cannot be easily expressed as regular low-level operations and +# types. +# +# struct { +# struct pypy_stackref_s hdr; +# type1 var1; +# ... +# typeN varN; +# } ref; +# + + +class InlineStackFrameworkGCTransformer(FrameworkGCTransformer): + + def push_roots(self, hop, keep_current_args=False): + livevars = self.get_livevars_for_roots(hop, keep_current_args) + hop.genop("gc_push_roots", livevars) + return livevars + + def pop_roots(self, hop, livevars): + pass + + def build_root_walker(self): + return InlineStackRootWalker(self) + +## def gct_direct_call(self, hop): +## fnptr = hop.spaceop.args[0].value +## try: +## close_stack = fnptr._obj._callable._gctransformer_hint_close_stack_ +## except AttributeError: +## close_stack = False +## if close_stack: +## self.handle_call_with_close_stack(hop) +## else: +## FrameworkGCTransformer.gct_direct_call(self, hop) + +## def handle_call_with_close_stack(self, hop): +## xxx + + +class InlineStackRootWalker(BaseRootWalker): + + def need_thread_support(self, gctransformer, getfn): + pass + + def walk_stack_roots(self, collect_stack_root): + gc = self.gc + stackref = get_pypy_stackref() + while stackref: + bitfield = stackref.unsigned[1] + index = 2 + while bitfield != 0: + if bitfield & 1: + addr = stackref + index * llmemory.sizeof(llmemory.Address) + if gc.points_to_valid_gc_object(addr): + collect_stack_root(gc, addr) + bitfield >>= 1 + index += 1 + stackref = stackref.address[0] + +get_pypy_stackref = rffi.llexternal('get_pypy_stackref', [], + llmemory.Address, + sandboxsafe=True, + _nowrapper=True) From arigo at codespeak.net Mon Dec 6 15:11:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 15:11:27 +0100 (CET) Subject: [pypy-svn] r79844 - pypy/trunk/pypy/module/cpyext/include Message-ID: <20101206141127.493F8282BAD@codespeak.net> Author: arigo Date: Mon Dec 6 15:11:24 2010 New Revision: 79844 Added: pypy/trunk/pypy/module/cpyext/include/fileobject.h Modified: pypy/trunk/pypy/module/cpyext/include/Python.h Log: Two small fixes to allow PIL to compile out of the box: * add some #included headers * add Py_FileSystemDefaultEncoding, always NULL for now Modified: pypy/trunk/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/trunk/pypy/module/cpyext/include/Python.h (original) +++ pypy/trunk/pypy/module/cpyext/include/Python.h Mon Dec 6 15:11:24 2010 @@ -8,6 +8,8 @@ # include # include # include +# include +# include # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_FUNC(RTYPE) RTYPE # define PyAPI_DATA(RTYPE) extern RTYPE @@ -103,6 +105,7 @@ #include "sliceobject.h" #include "datetime.h" #include "pystate.h" +#include "fileobject.h" // XXX This shouldn't be included here #include "structmember.h" Added: pypy/trunk/pypy/module/cpyext/include/fileobject.h ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/include/fileobject.h Mon Dec 6 15:11:24 2010 @@ -0,0 +1 @@ +#define Py_FileSystemDefaultEncoding NULL From arigo at codespeak.net Mon Dec 6 15:47:22 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 15:47:22 +0100 (CET) Subject: [pypy-svn] r79845 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20101206144722.59D6E282BAD@codespeak.net> Author: arigo Date: Mon Dec 6 15:47:20 2010 New Revision: 79845 Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Reproduce the bug of issue591. Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon Dec 6 15:47:20 2010 @@ -1954,6 +1954,50 @@ """ self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)', expected) + def test_bug_4(self): + ops = """ + [p0, p1, p2, p3, p4, i5, i6, p7, p8, p9] + + guard_value(i5, 0) [] + guard_nonnull(p9) [] + + guard_value(p3, 0) [] + + guard_class(p9, ConstClass(node_vtable)) [] + guard_class(p9, ConstClass(node_vtable)) [] + i14 = getfield_gc_pure(p9, descr=valuedescr) + i16 = int_lt(i14, 10000) + guard_true(i16) [] + + guard_nonnull(p9) [] + + p17 = getfield_gc(p0, descr=nextdescr) + guard_nonnull(p17) [] + guard_class(p17, 143782720) [] + guard_class(p17, 143782720) [] + guard_value(p17, ConstPtr(myptr)) [] + setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) + + guard_nonnull(p9) [] + + guard_class(p9, ConstClass(node_vtable)) [] + guard_class(p9, ConstClass(node_vtable)) [] + i26 = getfield_gc_pure(p9, descr=valuedescr) + i28 = int_add_ovf(i26, 1) + guard_no_overflow() [] + p30 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(p30, i28, descr=valuedescr) + + p37 = same_as(0) + i39 = same_as(0) + i41 = same_as(9) + p43 = same_as(0) + p45 = same_as(0) + jump(p0, p1, p2, p37, p4, i39, i41, p43, p45, p30) + """ + self.optimize_loop(ops, 'Not, Not, Not, Constant(0), Not, Constant(0), Not, Not, Not, ' + + 'Virtual(node_vtable, valuedescr=Not)', Ellipsis) + def test_invalid_loop_1(self): ops = """ [p1] From arigo at codespeak.net Mon Dec 6 15:57:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 15:57:10 +0100 (CET) Subject: [pypy-svn] r79846 - pypy/trunk/pypy/jit/metainterp/test Message-ID: <20101206145710.0A98D282BD4@codespeak.net> Author: arigo Date: Mon Dec 6 15:57:08 2010 New Revision: 79846 Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Reduce the test case. Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon Dec 6 15:57:08 2010 @@ -1956,47 +1956,12 @@ def test_bug_4(self): ops = """ - [p0, p1, p2, p3, p4, i5, i6, p7, p8, p9] - - guard_value(i5, 0) [] - guard_nonnull(p9) [] - - guard_value(p3, 0) [] - - guard_class(p9, ConstClass(node_vtable)) [] - guard_class(p9, ConstClass(node_vtable)) [] - i14 = getfield_gc_pure(p9, descr=valuedescr) - i16 = int_lt(i14, 10000) - guard_true(i16) [] - - guard_nonnull(p9) [] - - p17 = getfield_gc(p0, descr=nextdescr) - guard_nonnull(p17) [] - guard_class(p17, 143782720) [] - guard_class(p17, 143782720) [] - guard_value(p17, ConstPtr(myptr)) [] + [p9] setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) - - guard_nonnull(p9) [] - - guard_class(p9, ConstClass(node_vtable)) [] - guard_class(p9, ConstClass(node_vtable)) [] - i26 = getfield_gc_pure(p9, descr=valuedescr) - i28 = int_add_ovf(i26, 1) - guard_no_overflow() [] p30 = new_with_vtable(ConstClass(node_vtable)) - setfield_gc(p30, i28, descr=valuedescr) - - p37 = same_as(0) - i39 = same_as(0) - i41 = same_as(9) - p43 = same_as(0) - p45 = same_as(0) - jump(p0, p1, p2, p37, p4, i39, i41, p43, p45, p30) + jump(p30) """ - self.optimize_loop(ops, 'Not, Not, Not, Constant(0), Not, Constant(0), Not, Not, Not, ' + - 'Virtual(node_vtable, valuedescr=Not)', Ellipsis) + self.optimize_loop(ops, 'Virtual(node_vtable)', Ellipsis) def test_invalid_loop_1(self): ops = """ From arigo at codespeak.net Mon Dec 6 16:01:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 16:01:53 +0100 (CET) Subject: [pypy-svn] r79847 - in pypy/trunk/pypy/jit/metainterp: . test Message-ID: <20101206150153.A69E0282BDD@codespeak.net> Author: arigo Date: Mon Dec 6 16:01:51 2010 New Revision: 79847 Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Log: Fix: optimizefindnode did not consider that constants were escaped! Modified: pypy/trunk/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizefindnode.py Mon Dec 6 16:01:51 2010 @@ -128,6 +128,7 @@ assert isinstance(constbox, Const) node = InstanceNode() node.unique = UNIQUE_NO + node.escaped = True node.knownvaluebox = constbox self.nodes[box] = node return node Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Mon Dec 6 16:01:51 2010 @@ -1957,11 +1957,11 @@ def test_bug_4(self): ops = """ [p9] - setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) p30 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) jump(p30) """ - self.optimize_loop(ops, 'Virtual(node_vtable)', Ellipsis) + self.optimize_loop(ops, 'Not', ops) def test_invalid_loop_1(self): ops = """ From arigo at codespeak.net Mon Dec 6 16:03:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 16:03:08 +0100 (CET) Subject: [pypy-svn] r79848 - pypy/extradoc/planning Message-ID: <20101206150308.6A96C282BE0@codespeak.net> Author: arigo Date: Mon Dec 6 16:03:07 2010 New Revision: 79848 Modified: pypy/extradoc/planning/1.4.1.txt Log: Fixed. Modified: pypy/extradoc/planning/1.4.1.txt ============================================================================== --- pypy/extradoc/planning/1.4.1.txt (original) +++ pypy/extradoc/planning/1.4.1.txt Mon Dec 6 16:03:07 2010 @@ -11,10 +11,14 @@ * Fix a corner case in the GC (minimark). +* Fix cpyext on Mac OS/X. (Loading C extension modules in PyPy is still + considered *alpha stage!*) + +* Fixed a corner case in the JIT, leading to "Fatal RPython error: + AssertionError". -Plan: -* Fix bug in optimizeopt +Plan: * Out of line guards (?) From arigo at codespeak.net Mon Dec 6 16:39:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 16:39:13 +0100 (CET) Subject: [pypy-svn] r79849 - in pypy/trunk/pypy/module/cpyext: . src test Message-ID: <20101206153913.98EEB282BDD@codespeak.net> Author: arigo Date: Mon Dec 6 16:39:11 2010 New Revision: 79849 Modified: pypy/trunk/pypy/module/cpyext/intobject.py pypy/trunk/pypy/module/cpyext/src/getargs.c pypy/trunk/pypy/module/cpyext/test/test_intobject.py Log: Allow calls to PyInt_AsLong(NULL), and raise TypeError. Modified: pypy/trunk/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/intobject.py (original) +++ pypy/trunk/pypy/module/cpyext/intobject.py Mon Dec 6 16:39:11 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, build_type_checkers, Py_ssize_t) @@ -19,6 +20,9 @@ already one, and then return its value. If there is an error, -1 is returned, and the caller should check PyErr_Occurred() to find out whether there was an error, or whether the value just happened to be -1.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Unsigned, error=-1) @@ -26,6 +30,9 @@ """Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.uint_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Signed, error=CANNOT_FAIL) @@ -39,6 +46,9 @@ PyLongObject, if it is not already one, and then return its value as Py_ssize_t. """ + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(w_obj) # XXX this is wrong on win64 @cpython_api([Py_ssize_t], PyObject) @@ -48,4 +58,3 @@ returned. """ return space.wrap(ival) # XXX this is wrong on win64 - Modified: pypy/trunk/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/src/getargs.c (original) +++ pypy/trunk/pypy/module/cpyext/src/getargs.c Mon Dec 6 16:39:11 2010 @@ -445,18 +445,22 @@ for (i = 0; i < n; i++) { char *msg; PyObject *item; - item = PySequence_GetItem(arg, i); + /* CPython uses PySequence_GetItem() and Py_XDECREF() here, + exposing a crash (see http://bugs.python.org/issue6083). + It always crashes with PyPy, so we apply the fix being + discussed: we only allow a tuple. */ + item = PyTuple_GetItem(arg, i); if (item == NULL) { PyErr_Clear(); levels[0] = i+1; levels[1] = 0; - strncpy(msgbuf, "is not retrievable", bufsize); + strncpy(msgbuf, "is not retrievable (subargument " + "must be a real tuple with PyPy)", + bufsize); return msgbuf; } msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); - /* PySequence_GetItem calls tp->sq_item, which INCREFs */ - Py_XDECREF(item); if (msg != NULL) { levels[0] = i+1; return msg; Modified: pypy/trunk/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/trunk/pypy/module/cpyext/test/test_intobject.py Mon Dec 6 16:39:11 2010 @@ -19,6 +19,10 @@ assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + assert api.PyInt_AsLong(None) == -1 + assert api.PyErr_Occurred() is space.w_TypeError + api.PyErr_Clear() + assert api.PyInt_AsUnsignedLong(space.wrap(sys.maxint)) == sys.maxint assert api.PyInt_AsUnsignedLong(space.wrap(-5)) == sys.maxint * 2 + 1 assert api.PyErr_Occurred() is space.w_ValueError From cfbolz at codespeak.net Mon Dec 6 17:14:35 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 6 Dec 2010 17:14:35 +0100 (CET) Subject: [pypy-svn] r79850 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20101206161435.60A55282BDD@codespeak.net> Author: cfbolz Date: Mon Dec 6 17:14:33 2010 New Revision: 79850 Modified: pypy/trunk/pypy/objspace/std/longobject.py pypy/trunk/pypy/objspace/std/strutil.py pypy/trunk/pypy/objspace/std/test/test_strutil.py Log: Clean up interp_string_to_float to no longer use W_LongObjects for the computations that are needed in there. Instead just use rbigint directly. This makes it possible to remove some methods in longobject.py. Also, the function no longer needs its space argument, will remove that next. Modified: pypy/trunk/pypy/objspace/std/longobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/longobject.py (original) +++ pypy/trunk/pypy/objspace/std/longobject.py Mon Dec 6 17:14:33 2010 @@ -45,19 +45,6 @@ fromrarith_int._annspecialcase_ = "specialize:argtype(0)" fromrarith_int = staticmethod(fromrarith_int) - def fromdecimalstr(s): - return W_LongObject(rbigint.fromdecimalstr(s)) - fromdecimalstr = staticmethod(fromdecimalstr) - - def _count_bits(self): - return self.num._count_bits() - - def is_odd(self): - return self.num.is_odd() - - def get_sign(self): - return self.num.sign - registerimplementation(W_LongObject) # bool-to-long Modified: pypy/trunk/pypy/objspace/std/strutil.py ============================================================================== --- pypy/trunk/pypy/objspace/std/strutil.py (original) +++ pypy/trunk/pypy/objspace/std/strutil.py Mon Dec 6 17:14:33 2010 @@ -162,10 +162,9 @@ s = strip_spaces(s) if not s: - raise OperationError(space.w_ValueError, space.wrap( - "empty string for float()")) + raise ParseStringError("empty string for float()") + - low = s.lower() if low == "-inf": return -INFINITY @@ -204,68 +203,56 @@ if len(digits) == 0: digits = '0' - # a few abbreviations - from pypy.objspace.std import longobject - mklong = longobject.W_LongObject.fromint - d2long = longobject.W_LongObject.fromdecimalstr - adlong = longobject.add__Long_Long - longup = longobject.pow__Long_Long_None - multip = longobject.mul__Long_Long - divide = longobject.div__Long_Long - lshift = longobject.lshift__Long_Long - rshift = longobject.rshift__Long_Long - # 4) compute the exponent and truncate to +-400 if not exponent: exponent = '0' - w_le = d2long(exponent) - w_le = adlong(space, w_le, mklong(space, dexp)) + long_exponent = rbigint.fromdecimalstr(exponent) + long_exponent = long_exponent.add(rbigint.fromint(dexp)) try: - e = w_le.toint() + e = long_exponent.toint() except OverflowError: # XXX poking at internals - e = w_le.num.sign * 400 - if e >= 400: - e = 400 - elif e <= -400: - e = -400 + e = long_exponent.sign * 400 + else: + if e >= 400: + e = 400 + elif e <= -400: + e = -400 # 5) compute the value using long math and proper rounding. - w_lr = d2long(digits) - w_10 = mklong(space, 10) - w_1 = mklong(space, 1) + b_digits = rbigint.fromdecimalstr(digits) + b_10 = rbigint.fromint(10) + b_1 = rbigint.fromint(1) if e >= 0: bits = 0 - w_pten = longup(space, w_10, mklong(space, e), space.w_None) - w_m = multip(space, w_lr, w_pten) + b_power_of_ten = b_10.pow(rbigint.fromint(e)) + b_mantissa = b_digits.mul(b_power_of_ten) else: # compute a sufficiently large scale prec = MANTISSA_DIGITS * 2 + 22 # 128, maybe bits = - (int(math.ceil(-e / math.log10(2.0) - 1e-10)) + prec) - w_scale = lshift(space, w_1, mklong(space, -bits)) - w_pten = longup(space, w_10, mklong(space, -e), None) - w_tmp = multip(space, w_lr, w_scale) - w_m = divide(space, w_tmp, w_pten) + b_scale = b_1.lshift(-bits) + b_power_of_ten = b_10.pow(rbigint.fromint(-e)) + b_mantissa = b_digits.mul(b_scale).div(b_power_of_ten) # we now have a fairly large mantissa. # Shift it and round the last bit. # first estimate the bits and do a big shift - mbits = w_m._count_bits() + mbits = b_mantissa._count_bits() needed = MANTISSA_BITS if mbits > needed: if mbits > needed+1: shifted = mbits - (needed+1) - w_m = rshift(space, w_m, mklong(space, shifted)) + b_mantissa = b_mantissa.rshift(shifted) bits += shifted # do the rounding bits += 1 - round = w_m.is_odd() - w_m = rshift(space, w_m, w_1) - w_m = adlong(space, w_m, mklong(space, round)) + round = b_mantissa.is_odd() + b_mantissa = b_mantissa.rshift(1).add(rbigint.fromint(round)) try: - r = math.ldexp(w_m.tofloat(), bits) + r = math.ldexp(b_mantissa.tofloat(), bits) # XXX I guess we do not check for overflow in ldexp as we agreed to! if r == 2*r and r != 0.0: raise OverflowError Modified: pypy/trunk/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_strutil.py Mon Dec 6 17:14:33 2010 @@ -180,3 +180,4 @@ print repr(s) if s.strip(): # empty s raises OperationError directly py.test.raises(ParseStringError, string_to_float, s) + py.test.raises(ParseStringError, string_to_float, "") From cfbolz at codespeak.net Mon Dec 6 17:20:25 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 6 Dec 2010 17:20:25 +0100 (CET) Subject: [pypy-svn] r79851 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20101206162025.BABC8282BDD@codespeak.net> Author: cfbolz Date: Mon Dec 6 17:20:24 2010 New Revision: 79851 Modified: pypy/trunk/pypy/objspace/std/complextype.py pypy/trunk/pypy/objspace/std/floattype.py pypy/trunk/pypy/objspace/std/strutil.py pypy/trunk/pypy/objspace/std/test/test_strutil.py Log: remove unused space argument Modified: pypy/trunk/pypy/objspace/std/complextype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/complextype.py (original) +++ pypy/trunk/pypy/objspace/std/complextype.py Mon Dec 6 17:20:24 2010 @@ -1,7 +1,7 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.strutil import interp_string_to_float, ParseStringError +from pypy.objspace.std.strutil import string_to_float, ParseStringError from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod @@ -131,8 +131,8 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) try: - realval = interp_string_to_float(space, realstr) - imagval = interp_string_to_float(space, imagstr) + realval = string_to_float(realstr) + imagval = string_to_float(imagstr) except ParseStringError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) else: Modified: pypy/trunk/pypy/objspace/std/floattype.py ============================================================================== --- pypy/trunk/pypy/objspace/std/floattype.py (original) +++ pypy/trunk/pypy/objspace/std/floattype.py Mon Dec 6 17:20:24 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.strutil import ParseStringError -from pypy.objspace.std.strutil import interp_string_to_float +from pypy.objspace.std.strutil import string_to_float def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject @@ -10,7 +10,7 @@ if space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) @@ -21,7 +21,7 @@ from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) Modified: pypy/trunk/pypy/objspace/std/strutil.py ============================================================================== --- pypy/trunk/pypy/objspace/std/strutil.py (original) +++ pypy/trunk/pypy/objspace/std/strutil.py Mon Dec 6 17:20:24 2010 @@ -150,7 +150,7 @@ del calc_mantissa_bits MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1 -def interp_string_to_float(space, s): +def string_to_float(s): """ Conversion of string to float. This version tries to only raise on invalid literals. Modified: pypy/trunk/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_strutil.py Mon Dec 6 17:20:24 2010 @@ -131,8 +131,6 @@ assert string_to_bigint('1891234174197319').tolong() == 1891234174197319 def test_string_to_float(self): - def string_to_float(x): - return interp_string_to_float(self.space, x) assert string_to_float('0') == 0.0 assert string_to_float('1') == 1.0 assert string_to_float('-1.5') == -1.5 From afa at codespeak.net Mon Dec 6 17:27:37 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 6 Dec 2010 17:27:37 +0100 (CET) Subject: [pypy-svn] r79852 - in pypy/branch/fast-forward: . lib-python/modified-2.5.2/distutils pypy pypy/doc/config pypy/interpreter pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/optimizeopt pypy/jit/metainterp/test pypy/module/_stackless pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/src pypy/module/cpyext/test pypy/module/pypyjit/test pypy/module/sys pypy/rlib pypy/rlib/test pypy/rpython/lltypesystem pypy/tool/release pypy/translator/goal pypy/translator/platform Message-ID: <20101206162737.BDC69282BE3@codespeak.net> Author: afa Date: Mon Dec 6 17:27:34 2010 New Revision: 79852 Added: pypy/branch/fast-forward/pypy/module/cpyext/include/fileobject.h - copied unchanged from r79849, pypy/trunk/pypy/module/cpyext/include/fileobject.h Modified: pypy/branch/fast-forward/ (props changed) pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/unixccompiler.py pypy/branch/fast-forward/pypy/ (props changed) pypy/branch/fast-forward/pypy/doc/config/objspace.usemodules.array.txt (props changed) pypy/branch/fast-forward/pypy/interpreter/executioncontext.py pypy/branch/fast-forward/pypy/jit/codewriter/regalloc.py pypy/branch/fast-forward/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/fast-forward/pypy/jit/metainterp/optimizefindnode.py pypy/branch/fast-forward/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py pypy/branch/fast-forward/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/fast-forward/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/fast-forward/pypy/module/array/test/test_array.py pypy/branch/fast-forward/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/fast-forward/pypy/module/cpyext/include/Python.h pypy/branch/fast-forward/pypy/module/cpyext/intobject.py pypy/branch/fast-forward/pypy/module/cpyext/object.py pypy/branch/fast-forward/pypy/module/cpyext/pyerrors.py pypy/branch/fast-forward/pypy/module/cpyext/pythonrun.py pypy/branch/fast-forward/pypy/module/cpyext/src/getargs.c pypy/branch/fast-forward/pypy/module/cpyext/state.py pypy/branch/fast-forward/pypy/module/cpyext/stubs.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_intobject.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_object.py pypy/branch/fast-forward/pypy/module/cpyext/test/test_pyerrors.py pypy/branch/fast-forward/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/fast-forward/pypy/module/sys/vm.py pypy/branch/fast-forward/pypy/rlib/rerased.py (props changed) pypy/branch/fast-forward/pypy/rlib/test/test_rerased.py (props changed) pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/fast-forward/pypy/tool/release/package.py pypy/branch/fast-forward/pypy/translator/goal/app_main.py pypy/branch/fast-forward/pypy/translator/platform/darwin.py Log: Merge from trunk: -r79790:79849 Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/unixccompiler.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/unixccompiler.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/distutils/unixccompiler.py Mon Dec 6 17:27:34 2010 @@ -121,7 +121,22 @@ } if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() executables['ranlib'] = ["ranlib"] + executables['linker_so'] += ['-undefined', 'dynamic_lookup'] + + for k, v in executables.iteritems(): + if v and v[0] == 'cc': + v += ['-arch', arch] + # Needed for the filename generation methods provided by the base # class, CCompiler. NB. whoever instantiates/uses a particular Modified: pypy/branch/fast-forward/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/executioncontext.py Mon Dec 6 17:27:34 2010 @@ -27,7 +27,6 @@ def __init__(self, space): self.space = space self.topframeref = jit.vref_None - self.framestackdepth = 0 # tracing: space.frame_trace_action.fire() must be called to ensure # that tracing occurs whenever self.w_tracefunc or self.is_tracing # is modified. @@ -54,9 +53,6 @@ return frame def enter(self, frame): - if self.framestackdepth > self.space.sys.recursionlimit: - raise self.space.prebuilt_recursion_error - self.framestackdepth += 1 frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) @@ -66,7 +62,6 @@ self._trace(frame, 'leaveframe', self.space.w_None) finally: self.topframeref = frame.f_backref - self.framestackdepth -= 1 jit.virtual_ref_finish(frame) if self.w_tracefunc is not None and not frame.hide(): @@ -80,7 +75,6 @@ def __init__(self): self.topframe = None - self.framestackdepth = 0 self.w_tracefunc = None self.profilefunc = None self.w_profilefuncarg = None @@ -88,7 +82,6 @@ def enter(self, ec): ec.topframeref = jit.non_virtual_ref(self.topframe) - ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc ec.w_profilefuncarg = self.w_profilefuncarg @@ -97,7 +90,6 @@ def leave(self, ec): self.topframe = ec.gettopframe() - self.framestackdepth = ec.framestackdepth self.w_tracefunc = ec.w_tracefunc self.profilefunc = ec.profilefunc self.w_profilefuncarg = ec.w_profilefuncarg @@ -105,7 +97,6 @@ def clear_framestack(self): self.topframe = None - self.framestackdepth = 0 # the following interface is for pickling and unpickling def getstate(self, space): @@ -121,17 +112,14 @@ self.topframe = space.interp_w(PyFrame, frames_w[-1]) else: self.topframe = None - self.framestackdepth = len(frames_w) def getframestack(self): - index = self.framestackdepth - lst = [None] * index + lst = [] f = self.topframe - while index > 0: - index -= 1 - lst[index] = f + while f is not None: + lst.append(f) f = f.f_backref() - assert f is None + lst.reverse() return lst # coroutine: I think this is all, folks! Modified: pypy/branch/fast-forward/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/regalloc.py Mon Dec 6 17:27:34 2010 @@ -36,7 +36,7 @@ if isinstance(v1, Variable): die_at[v1] = i if op.result is not None: - die_at[op.result] = i + die_at[op.result] = i + 1 if isinstance(block.exitswitch, tuple): for x in block.exitswitch: die_at.pop(x, None) Modified: pypy/branch/fast-forward/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/codewriter/test/test_regalloc.py Mon Dec 6 17:27:34 2010 @@ -281,22 +281,22 @@ # this used to produce bogus code, containing these two # lines in the following broken order: # last_exc_value -> %r0 - # ref_copy %r0 -> %r2 -- but expect to read the old value of %r0! + # ref_copy %r0 -> %r1 -- but expect to read the old value of %r0! self.check_assembler(graph, """ residual_call_r_r $<* fn bar>, , R[%r0] -> %r1 -live- - residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r2 + residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r1 -live- catch_exception L1 - ref_return %r2 + ref_return %r1 --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 - ref_copy %r0 -> %r2 + ref_copy %r0 -> %r1 last_exc_value -> %r0 residual_call_r_r $<* fn foo>, , R[%r0] -> %r0 -live- - ref_return %r2 + ref_return %r1 --- L2: reraise Modified: pypy/branch/fast-forward/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/optimizefindnode.py Mon Dec 6 17:27:34 2010 @@ -128,6 +128,7 @@ assert isinstance(constbox, Const) node = InstanceNode() node.unique = UNIQUE_NO + node.escaped = True node.knownvaluebox = constbox self.nodes[box] = node return node Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizeopt.py Mon Dec 6 17:27:34 2010 @@ -1954,6 +1954,15 @@ """ self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)', expected) + def test_bug_4(self): + ops = """ + [p9] + p30 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) + jump(p30) + """ + self.optimize_loop(ops, 'Not', ops) + def test_invalid_loop_1(self): ops = """ [p1] Modified: pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/fast-forward/pypy/module/_stackless/interp_coroutine.py Mon Dec 6 17:27:34 2010 @@ -304,16 +304,19 @@ def w_descr__framestack(space, self): assert isinstance(self, AppCoroutine) - index = self.subctx.framestackdepth - if not index: - return space.newtuple([]) - items = [None] * index + counter = 0 f = self.subctx.topframe - while index > 0: - index -= 1 - items[index] = space.wrap(f) + while f is not None: + counter += 1 f = f.f_backref() - assert f is None + items = [None] * counter + f = self.subctx.topframe + while f is not None: + counter -= 1 + assert counter >= 0 + items[counter] = space.wrap(f) + f = f.f_backref() + assert counter == 0 return space.newtuple(items) def makeStaticMethod(module, classname, funcname): Modified: pypy/branch/fast-forward/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/array/test/test_array.py (original) +++ pypy/branch/fast-forward/pypy/module/array/test/test_array.py Mon Dec 6 17:27:34 2010 @@ -65,6 +65,7 @@ raises(TypeError, self.array, tc, None) def test_value_range(self): + import sys values = (-129, 128, -128, 127, 0, 255, -1, 256, -32768, 32767, -32769, 32768, 65535, 65536, -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, @@ -89,7 +90,12 @@ a.append(v) for i, v in enumerate(ok * 2): assert a[i] == v - assert type(a[i]) is pt + assert type(a[i]) is pt or ( + # A special case: we return ints in Array('I') on 64-bits, + # whereas CPython returns longs. The difference is + # probably acceptable. + tc == 'I' and + sys.maxint > 2147483647 and type(a[i]) is int) for v in ok: a[1] = v assert a[0] == ok[0] Modified: pypy/branch/fast-forward/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/include/Python.h Mon Dec 6 17:27:34 2010 @@ -8,6 +8,8 @@ # include # include # include +# include +# include # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_FUNC(RTYPE) RTYPE # define PyAPI_DATA(RTYPE) extern RTYPE @@ -115,6 +117,7 @@ #include "sliceobject.h" #include "datetime.h" #include "pystate.h" +#include "fileobject.h" // XXX This shouldn't be included here #include "structmember.h" @@ -132,4 +135,8 @@ #define PyDoc_STR(str) "" #endif +/* PyPy does not implement --with-fpectl */ +#define PyFPE_START_PROTECT(err_string, leave_stmt) +#define PyFPE_END_PROTECT(v) + #endif Modified: pypy/branch/fast-forward/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/intobject.py Mon Dec 6 17:27:34 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import ( cpython_api, build_type_checkers, PyObject, CONST_STRING, CANNOT_FAIL, Py_ssize_t) @@ -20,6 +21,9 @@ already one, and then return its value. If there is an error, -1 is returned, and the caller should check PyErr_Occurred() to find out whether there was an error, or whether the value just happened to be -1.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Unsigned, error=-1) @@ -27,6 +31,9 @@ """Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.uint_w(space.int(w_obj)) @cpython_api([PyObject], rffi.ULONG, error=-1) @@ -54,6 +61,9 @@ PyLongObject, if it is not already one, and then return its value as Py_ssize_t. """ + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(w_obj) # XXX this is wrong on win64 @cpython_api([Py_ssize_t], PyObject) @@ -85,4 +95,3 @@ if pend: pend[0] = rffi.ptradd(str, len(s)) return space.call_function(space.w_int, w_str, w_base) - Modified: pypy/branch/fast-forward/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/object.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/object.py Mon Dec 6 17:27:34 2010 @@ -2,12 +2,13 @@ from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, - Py_GE, CONST_STRING, FILEP, fwrite) + Py_GE, CONST_STRING, FILEP, fwrite, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference, get_typedescr, RefcountState) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall +from pypy.module._file.interp_file import W_File from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject from pypy.interpreter.error import OperationError @@ -428,6 +429,34 @@ rffi.free_nonmovingbuffer(data, buf) return 0 +PyFile_Check, PyFile_CheckExact = build_type_checkers("File", W_File) + + at cpython_api([PyObject, rffi.INT_real], PyObject) +def PyFile_GetLine(space, w_obj, n): + """ + Equivalent to p.readline([n]), this function reads one line from the + object p. p may be a file object or any object with a readline() + method. If n is 0, exactly one line is read, regardless of the length of + the line. If n is greater than 0, no more than n bytes will be read + from the file; a partial line can be returned. In both cases, an empty string + is returned if the end of the file is reached immediately. If n is less than + 0, however, one line is read regardless of length, but EOFError is + raised if the end of the file is reached immediately.""" + try: + w_readline = space.getattr(w_obj, space.wrap('readline')) + except OperationError: + raise OperationError( + space.w_TypeError, space.wrap( + "argument must be a file, or have a readline() method.")) + + n = rffi.cast(lltype.Signed, n) + if space.is_true(space.gt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline, space.wrap(n)) + elif space.is_true(space.lt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline) + else: + # XXX Raise EOFError as specified + return space.call_function(w_readline) @cpython_api([CONST_STRING, CONST_STRING], PyObject) def PyFile_FromString(space, filename, mode): """ @@ -437,4 +466,3 @@ w_filename = space.wrap(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) - Modified: pypy/branch/fast-forward/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/pyerrors.py Mon Dec 6 17:27:34 2010 @@ -5,7 +5,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, Py_DecRef, borrow_from) + PyObject, PyObjectP, make_ref, from_ref, Py_DecRef, borrow_from) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from pypy.rlib.rposix import get_errno @@ -80,6 +80,21 @@ Py_DecRef(space, w_value) Py_DecRef(space, w_traceback) + at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) +def PyErr_NormalizeException(space, exc_p, val_p, tb_p): + """Under certain circumstances, the values returned by PyErr_Fetch() below + can be "unnormalized", meaning that *exc is a class object but *val is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance.""" + operr = OperationError(from_ref(space, exc_p[0]), + from_ref(space, val_p[0])) + operr.normalize_exception(space) + Py_DecRef(space, exc_p[0]) + Py_DecRef(space, val_p[0]) + exc_p[0] = make_ref(space, operr.w_type) + val_p[0] = make_ref(space, operr.get_w_value(space)) + @cpython_api([], lltype.Void) def PyErr_BadArgument(space): """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where @@ -114,10 +129,29 @@ function around a system call can write return PyErr_SetFromErrno(type); when the system call returns an error. Return value: always NULL.""" + PyErr_SetFromErrnoWithFilename(space, w_type, + lltype.nullptr(rffi.CCHARP.TO)) + + at cpython_api([PyObject, rffi.CCHARP], PyObject) +def PyErr_SetFromErrnoWithFilename(space, w_type, llfilename): + """Similar to PyErr_SetFromErrno(), with the additional behavior that if + filename is not NULL, it is passed to the constructor of type as a third + parameter. In the case of exceptions such as IOError and OSError, + this is used to define the filename attribute of the exception instance. + Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() msg = os.strerror(errno) - w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + if llfilename: + w_filename = rffi.charp2str(llfilename) + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg), + space.wrap(w_filename)) + else: + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg)) raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) Modified: pypy/branch/fast-forward/pypy/module/cpyext/pythonrun.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/pythonrun.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/pythonrun.py Mon Dec 6 17:27:34 2010 @@ -1,6 +1,16 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.state import State @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def Py_IsInitialized(space): return 1 + + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) +def Py_GetProgramName(space): + """ + Return the program name set with Py_SetProgramName(), or the default. + The returned string points into static storage; the caller should not modify its + value.""" + return space.fromcache(State).get_programname() + Modified: pypy/branch/fast-forward/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/src/getargs.c Mon Dec 6 17:27:34 2010 @@ -445,18 +445,22 @@ for (i = 0; i < n; i++) { char *msg; PyObject *item; - item = PySequence_GetItem(arg, i); + /* CPython uses PySequence_GetItem() and Py_XDECREF() here, + exposing a crash (see http://bugs.python.org/issue6083). + It always crashes with PyPy, so we apply the fix being + discussed: we only allow a tuple. */ + item = PyTuple_GetItem(arg, i); if (item == NULL) { PyErr_Clear(); levels[0] = i+1; levels[1] = 0; - strncpy(msgbuf, "is not retrievable", bufsize); + strncpy(msgbuf, "is not retrievable (subargument " + "must be a real tuple with PyPy)", + bufsize); return msgbuf; } msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); - /* PySequence_GetItem calls tp->sq_item, which INCREFs */ - Py_XDECREF(item); if (msg != NULL) { levels[0] = i+1; return msg; Modified: pypy/branch/fast-forward/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/state.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/state.py Mon Dec 6 17:27:34 2010 @@ -9,6 +9,7 @@ def __init__(self, space): self.space = space self.reset() + self.programname = lltype.nullptr(rffi.CCHARP.TO) def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef @@ -82,3 +83,16 @@ for func in INIT_FUNCTIONS: func(space) self.check_and_raise_exception() + + def get_programname(self): + if not self.programname: + space = self.space + argv = space.sys.get('argv') + if space.int_w(space.len(argv)): + argv0 = space.getitem(argv, space.wrap(0)) + progname = space.str_w(argv0) + else: + progname = "pypy" + self.programname = rffi.str2charp(progname) + lltype.render_immortal(self.programname) + return self.programname Modified: pypy/branch/fast-forward/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/stubs.py Mon Dec 6 17:27:34 2010 @@ -597,24 +597,6 @@ """ raise NotImplementedError - at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) -def PyErr_NormalizeException(space, exc, val, tb): - """Under certain circumstances, the values returned by PyErr_Fetch() below - can be "unnormalized", meaning that *exc is a class object but *val is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], PyObject) -def PyErr_SetFromErrnoWithFilename(space, type, filename): - """Similar to PyErr_SetFromErrno(), with the additional behavior that if - filename is not NULL, it is passed to the constructor of type as a third - parameter. In the case of exceptions such as IOError and OSError, - this is used to define the filename attribute of the exception instance. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([rffi.INT_real], PyObject) def PyErr_SetFromWindowsErr(space, ierr): """This is a convenience function to raise WindowsError. If called with @@ -723,21 +705,6 @@ successful invocation of Py_EnterRecursiveCall().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_Check(space, p): - """Return true if its argument is a PyFileObject or a subtype of - PyFileObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_CheckExact(space, p): - """Return true if its argument is a PyFileObject, but not a subtype of - PyFileObject. - """ - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file @@ -771,22 +738,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], PyObject) -def PyFile_GetLine(space, p, n): - """ - - - - Equivalent to p.readline([n]), this function reads one line from the - object p. p may be a file object or any object with a readline() - method. If n is 0, exactly one line is read, regardless of the length of - the line. If n is greater than 0, no more than n bytes will be read - from the file; a partial line can be returned. In both cases, an empty string - is returned if the end of the file is reached immediately. If n is less than - 0, however, one line is read regardless of length, but EOFError is - raised if the end of the file is reached immediately.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyFile_Name(space, p): """Return the name of the file specified by p as a string object.""" @@ -1310,17 +1261,6 @@ raise NotImplementedError @cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) -def Py_GetProgramName(space, ): - """ - - - - Return the program name set with Py_SetProgramName(), or the default. - The returned string points into static storage; the caller should not modify its - value.""" - raise NotImplementedError - - at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPrefix(space, ): """Return the prefix for installed platform-independent files. This is derived through a number of complicated rules from the program name set with Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_cpyext.py Mon Dec 6 17:27:34 2010 @@ -691,3 +691,19 @@ assert mod.get_names() == ('cell', 'module', 'property', 'staticmethod', 'builtin_function_or_method') + + def test_get_programname(self): + mod = self.import_extension('foo', [ + ('get_programname', 'METH_NOARGS', + ''' + char* name1 = Py_GetProgramName(); + char* name2 = Py_GetProgramName(); + if (name1 != name2) + Py_RETURN_FALSE; + return PyString_FromString(name1); + ''' + ), + ]) + p = mod.get_programname() + print p + assert 'py' in p Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_intobject.py Mon Dec 6 17:27:34 2010 @@ -20,6 +20,10 @@ assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + assert api.PyInt_AsLong(None) == -1 + assert api.PyErr_Occurred() is space.w_TypeError + api.PyErr_Clear() + assert api.PyInt_AsUnsignedLong(space.wrap(sys.maxint)) == sys.maxint assert api.PyInt_AsUnsignedLong(space.wrap(-5)) == sys.maxint * 2 + 1 assert api.PyErr_Occurred() is space.w_ValueError Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_object.py Mon Dec 6 17:27:34 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ - Py_GE, Py_GT + Py_GE, Py_GT, fopen, fclose, fwrite from pypy.tool.udir import udir class TestObject(BaseApiTest): @@ -188,10 +188,45 @@ rffi.free_charp(filename) rffi.free_charp(mode) + assert api.PyFile_Check(w_file) + assert api.PyFile_CheckExact(w_file) + assert not api.PyFile_Check(space.wrap("text")) + space.call_method(w_file, "write", space.wrap("text")) space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" + def test_file_getline(self, space, api): + filename = rffi.str2charp(str(udir / "_test_file")) + + mode = rffi.str2charp("w") + w_file = api.PyFile_FromString(filename, mode) + space.call_method(w_file, "write", + space.wrap("line1\nline2\nline3\nline4")) + space.call_method(w_file, "close") + + rffi.free_charp(mode) + mode = rffi.str2charp("r") + w_file = api.PyFile_FromString(filename, mode) + rffi.free_charp(filename) + rffi.free_charp(mode) + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "line1\n" + + w_line = api.PyFile_GetLine(w_file, 4) + assert space.str_w(w_line) == "line" + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "2\n" + + # XXX We ought to raise an EOFError here, but don't + w_line = api.PyFile_GetLine(w_file, -1) + # assert api.PyErr_Occurred() is space.w_EOFError + assert space.str_w(w_line) == "line3\n" + + space.call_method(w_file, "close") + class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) Modified: pypy/branch/fast-forward/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/test/test_pyerrors.py Mon Dec 6 17:27:34 2010 @@ -129,6 +129,41 @@ ]) assert module.check_error() + + def test_normalize(self): + module = self.import_extension('foo', [ + ("check_error", "METH_NOARGS", + ''' + PyObject *type, *val, *tb; + PyErr_SetString(PyExc_TypeError, "message"); + + PyErr_Fetch(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (!PyString_Check(val)) + Py_RETURN_FALSE; + /* Normalize */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + /* Normalize again */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + PyErr_Restore(type, val, tb); + PyErr_Clear(); + Py_RETURN_TRUE; + ''' + ), + ]) + assert module.check_error() + def test_SetFromErrno(self): import sys if sys.platform != 'win32': @@ -149,3 +184,26 @@ except OSError, e: assert e.errno == errno.EBADF assert e.strerror == os.strerror(errno.EBADF) + assert e.filename == None + + def test_SetFromErrnoWithFilename(self): + import sys + if sys.platform != 'win32': + skip("callbacks through ll2ctypes modify errno") + import errno, os + + module = self.import_extension('foo', [ + ("set_from_errno", "METH_NOARGS", + ''' + errno = EBADF; + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "blyf"); + return NULL; + '''), + ], + prologue="#include ") + try: + module.set_from_errno() + except OSError, e: + assert e.filename == "blyf" + assert e.errno == errno.EBADF + assert e.strerror == os.strerror(errno.EBADF) Modified: pypy/branch/fast-forward/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/fast-forward/pypy/module/pypyjit/test/test_pypy_c.py Mon Dec 6 17:27:34 2010 @@ -885,11 +885,14 @@ def test_array_sum(self): for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): res = 19352859 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array @@ -937,11 +940,14 @@ print '='*65 print '='*20, 'running test for tc=%r' % (tc,), '='*20 res = 73574560 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array Modified: pypy/branch/fast-forward/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/vm.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/vm.py Mon Dec 6 17:27:34 2010 @@ -42,27 +42,21 @@ f = ec.getnextframe_nohidden(f) return space.wrap(f) -# directly from the C code in ceval.c, might be moved somewhere else. - def setrecursionlimit(space, w_new_limit): - """Set the maximum depth of the Python interpreter stack to n. This -limit prevents infinite recursion from causing an overflow of the C -stack and crashing Python. The highest possible limit is platform -dependent.""" + """DEPRECATED on PyPy. Will issue warning and not work + """ new_limit = space.int_w(w_new_limit) if new_limit <= 0: raise OperationError(space.w_ValueError, space.wrap("recursion limit must be positive")) # global recursion_limit # we need to do it without writing globals. + space.warn('setrecursionlimit deprecated', space.w_DeprecationWarning) space.sys.recursionlimit = new_limit def getrecursionlimit(space): - """Return the current value of the recursion limit, the maximum depth - of the Python interpreter stack. This limit prevents infinite - recursion from causing an overflow of the C stack and crashing Python. + """DEPRECATED on PyPy. Will issue warning and not work """ - return space.wrap(space.sys.recursionlimit) def setcheckinterval(space, interval): Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll2ctypes.py Mon Dec 6 17:27:34 2010 @@ -413,6 +413,7 @@ subcls = get_common_subclass(mixin_cls, instance.__class__) instance.__class__ = subcls instance._storage = ctypes_storage + assert ctypes_storage # null pointer? class _parentable_mixin(object): """Mixin added to _parentable containers when they become ctypes-based. Modified: pypy/branch/fast-forward/pypy/tool/release/package.py ============================================================================== --- pypy/branch/fast-forward/pypy/tool/release/package.py (original) +++ pypy/branch/fast-forward/pypy/tool/release/package.py Mon Dec 6 17:27:34 2010 @@ -78,7 +78,7 @@ old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) # ignore errors + os.system("strip -x " + str(archive_pypy_c)) # ignore errors if USE_TARFILE_MODULE: import tarfile tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') Modified: pypy/branch/fast-forward/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/goal/app_main.py (original) +++ pypy/branch/fast-forward/pypy/translator/goal/app_main.py Mon Dec 6 17:27:34 2010 @@ -383,8 +383,9 @@ cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 - # but we need more in the translated PyPy for the compiler package - sys.setrecursionlimit(5000) + # but we need more in the translated PyPy for the compiler package + if '__pypy__' not in sys.builtin_module_names: + sys.setrecursionlimit(5000) if unbuffered: set_unbuffered_io() Modified: pypy/branch/fast-forward/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/platform/darwin.py (original) +++ pypy/branch/fast-forward/pypy/translator/platform/darwin.py Mon Dec 6 17:27:34 2010 @@ -11,8 +11,10 @@ shared_only = () so_ext = 'so' - - default_cc = 'gcc' + + # NOTE: GCC 4.2 will fail at runtime due to subtle issues, possibly + # related to GC roots. Using LLVM-GCC or Clang will break the build. + default_cc = 'gcc-4.0' def __init__(self, cc=None): if cc is None: @@ -87,4 +89,3 @@ link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4') cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4') - default_cc = 'gcc-4.0' From afa at codespeak.net Mon Dec 6 17:28:24 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 6 Dec 2010 17:28:24 +0100 (CET) Subject: [pypy-svn] r79853 - in pypy/branch/fast-forward/lib-python/modified-2.7.0: . distutils Message-ID: <20101206162824.B403B282BE7@codespeak.net> Author: afa Date: Mon Dec 6 17:28:23 2010 New Revision: 79853 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/ (props changed) pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/unixccompiler.py Log: Merge 2.5.2 changes into 2.7.0 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/unixccompiler.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/unixccompiler.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/distutils/unixccompiler.py Mon Dec 6 17:28:23 2010 @@ -125,7 +125,22 @@ } if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() executables['ranlib'] = ["ranlib"] + executables['linker_so'] += ['-undefined', 'dynamic_lookup'] + + for k, v in executables.iteritems(): + if v and v[0] == 'cc': + v += ['-arch', arch] + # Needed for the filename generation methods provided by the base # class, CCompiler. NB. whoever instantiates/uses a particular From arigo at codespeak.net Mon Dec 6 18:20:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 6 Dec 2010 18:20:35 +0100 (CET) Subject: [pypy-svn] r79854 - in pypy/trunk/pypy/module/cpyext: . src Message-ID: <20101206172035.619B0282B90@codespeak.net> Author: arigo Date: Mon Dec 6 18:20:32 2010 New Revision: 79854 Added: pypy/trunk/pypy/module/cpyext/pypyintf.py Modified: pypy/trunk/pypy/module/cpyext/__init__.py pypy/trunk/pypy/module/cpyext/src/getargs.c Log: Rewrite getargs.c to use PyPy_Borrow(), a new function that register a borrowing dependency from C code. Modified: pypy/trunk/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/trunk/pypy/module/cpyext/__init__.py (original) +++ pypy/trunk/pypy/module/cpyext/__init__.py Mon Dec 6 18:20:32 2010 @@ -69,6 +69,7 @@ import pypy.module.cpyext.weakrefobject import pypy.module.cpyext.funcobject import pypy.module.cpyext.classobject +import pypy.module.cpyext.pypyintf # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Added: pypy/trunk/pypy/module/cpyext/pypyintf.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/module/cpyext/pypyintf.py Mon Dec 6 18:20:32 2010 @@ -0,0 +1,9 @@ +from pypy.module.cpyext.api import cpython_api +from pypy.module.cpyext.pyobject import PyObject, borrow_from + + + at cpython_api([PyObject, PyObject], PyObject) +def PyPy_Borrow(space, w_parentobj, w_obj): + """Returns a borrowed reference to 'obj', borrowing from the 'parentobj'. + """ + return borrow_from(w_parentobj, w_obj) Modified: pypy/trunk/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/trunk/pypy/module/cpyext/src/getargs.c (original) +++ pypy/trunk/pypy/module/cpyext/src/getargs.c Mon Dec 6 18:20:32 2010 @@ -445,22 +445,19 @@ for (i = 0; i < n; i++) { char *msg; PyObject *item; - /* CPython uses PySequence_GetItem() and Py_XDECREF() here, - exposing a crash (see http://bugs.python.org/issue6083). - It always crashes with PyPy, so we apply the fix being - discussed: we only allow a tuple. */ - item = PyTuple_GetItem(arg, i); + item = PySequence_GetItem(arg, i); if (item == NULL) { PyErr_Clear(); levels[0] = i+1; levels[1] = 0; - strncpy(msgbuf, "is not retrievable (subargument " - "must be a real tuple with PyPy)", - bufsize); + strncpy(msgbuf, "is not retrievable", bufsize); return msgbuf; } + PyPy_Borrow(arg, item); msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); + /* PySequence_GetItem calls tp->sq_item, which INCREFs */ + Py_XDECREF(item); if (msg != NULL) { levels[0] = i+1; return msg; From cfbolz at codespeak.net Mon Dec 6 21:08:47 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 6 Dec 2010 21:08:47 +0100 (CET) Subject: [pypy-svn] r79855 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20101206200847.35C11282B90@codespeak.net> Author: cfbolz Date: Mon Dec 6 21:08:13 2010 New Revision: 79855 Modified: pypy/trunk/pypy/objspace/std/complexobject.py pypy/trunk/pypy/objspace/std/test/test_complexobject.py Log: refactor the complexobject implementation to be less close to the C code and to be a bit more readable (imo). Modified: pypy/trunk/pypy/objspace/std/complexobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/complexobject.py (original) +++ pypy/trunk/pypy/objspace/std/complexobject.py Mon Dec 6 21:08:13 2010 @@ -23,90 +23,92 @@ """ representation for debugging purposes """ return "" % (w_self.realval, w_self.imagval) -registerimplementation(W_ComplexObject) - -c_1 = (1.0, 0.0) + def sub(self, other): + return W_ComplexObject(self.realval - other.realval, + self.imagval - other.imagval) + + def mul(self, other): + r = self.realval * other.realval - self.imagval * other.imagval + i = self.realval * other.imagval + self.imagval * other.realval + return W_ComplexObject(r, i) + + def div(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 < 0: + abs_r2 = - r2 + else: + abs_r2 = r2 + if i2 < 0: + abs_i2 = - i2 + else: + abs_i2 = i2 + if abs_r2 >= abs_i2: + if abs_r2 == 0.0: + raise ZeroDivisionError + else: + ratio = i2 / r2 + denom = r2 + i2 * ratio + rr = (r1 + i1 * ratio) / denom + ir = (i1 - r1 * ratio) / denom + else: + ratio = r2 / i2 + denom = r2 * ratio + i2 + assert i2 != 0.0 + rr = (r1 * ratio + i1) / denom + ir = (i1 * ratio - r1) / denom + return W_ComplexObject(rr,ir) + + def divmod(self, other): + w_div = self.div(other) + div = math.floor(w_div.realval) + w_mod = self.sub( + W_ComplexObject(other.realval * div, other.imagval * div)) + return (W_ComplexObject(div, 0), w_mod) + + def pow(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 == 0.0 and i2 == 0.0: + rr, ir = 1, 0 + elif r1 == 0.0 and i1 == 0.0: + if i2 != 0.0 or r2 < 0.0: + raise ZeroDivisionError + rr, ir = (0.0, 0.0) + else: + vabs = math.hypot(r1,i1) + len = math.pow(vabs,r2) + at = math.atan2(i1,r1) + phase = at * r2 + if i2 != 0.0: + len /= math.exp(at * i2) + phase += i2 * math.log(vabs) + rr = len * math.cos(phase) + ir = len * math.sin(phase) + return W_ComplexObject(rr, ir) + + def pow_int(self, n): + if n > 100 or n < -100: + return self.pow(W_ComplexObject(1.0 * n, 0.0)) + elif n > 0: + return self.pow_positive_int(n) + else: + return w_one.div(self.pow_positive_int(-n)) -def _sum(c1, c2): - return (c1[0]+c2[0],c1[1]+c2[1]) + def pow_positive_int(self, n): + mask = 1 + w_result = w_one + while mask > 0 and n >= mask: + if n & mask: + w_result = w_result.mul(self) + mask <<= 1 + self = self.mul(self) -def _diff(c1, c2): - return (c1[0]-c2[0],c1[1]-c2[1]) + return w_result -def _prod(c1, c2): - r = c1[0]*c2[0] - c1[1]*c2[1] - i = c1[0]*c2[1] + c1[1]*c2[0] - return (r,i) - -def _quot(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 < 0: - abs_r2 = - r2 - else: - abs_r2 = r2 - if i2 < 0: - abs_i2 = - i2 - else: - abs_i2 = i2 - if abs_r2 >= abs_i2: - if abs_r2 == 0.0: - raise ZeroDivisionError - else: - ratio = i2 / r2 - denom = r2 + i2 * ratio - rr = (r1 + i1 * ratio) / denom - ir = (i1 - r1 * ratio) / denom - else: - ratio = r2 / i2 - denom = r2 * ratio + i2 - assert i2 != 0.0 - rr = (r1 * ratio + i1) / denom - ir = (i1 * ratio - r1) / denom - return (rr,ir) - -def _pow(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 == 0.0 and i2 == 0.0: - rr, ir = c_1 - elif r1 == 0.0 and i1 == 0.0: - if i2 != 0.0 or r2 < 0.0: - raise ZeroDivisionError - rr, ir = (0.0, 0.0) - else: - vabs = math.hypot(r1,i1) - len = math.pow(vabs,r2) - at = math.atan2(i1,r1) - phase = at * r2 - if i2 != 0.0: - len /= math.exp(at * i2) - phase += i2 * math.log(vabs) - rr = len * math.cos(phase) - ir = len * math.sin(phase) - return (rr, ir) - -def _powu(c,n): - mask = 1; - rr, ir = c_1 - rp = c[0] - ip = c[1] - while mask > 0 and n >= mask: - if n & mask: - rr, ir = _prod((rr, ir), (rp, ip)) - mask <<= 1 - rp, ip = _prod((rp, ip), (rp, ip)) - - return (rr, ir) - -def _powi(c,n): - if n > 100 or n < -100: - return _pow(c,(1.0 * n, 0.0)) - elif n > 0: - return _powu(c, n) - else: - return _quot(c_1, _powu(c, -n)) +registerimplementation(W_ComplexObject) +w_one = W_ComplexObject(1, 0) def delegate_Bool2Complex(space, w_bool): @@ -126,38 +128,25 @@ return W_ComplexObject(w_float.floatval, 0.0) def hash__Complex(space, w_value): - #this is straight out of CPython complex implementation - hashreal = _hash_float(space, w_value.realval) - if hashreal == -1: - return space.newint(-1) hashimg = _hash_float(space, w_value.imagval) - if hashimg == -1: - return space.newint(-1) combined = hashreal + 1000003 * hashimg - if (combined == -1): - combined = -2 return space.newint(combined) -def _w2t(space, w_complex): - "convert an interplevel complex object to a tuple representation" - return w_complex.realval, w_complex.imagval - -def _t2w(space, c): - return W_ComplexObject(c[0], c[1]) - def add__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _sum(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval + w_complex2.realval, + w_complex1.imagval + w_complex2.imagval) def sub__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _diff(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval - w_complex2.realval, + w_complex1.imagval - w_complex2.imagval) def mul__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _prod(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.mul(w_complex2) def div__Complex_Complex(space, w_complex1, w_complex2): try: - return _t2w(space, _quot(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.div(w_complex2) except ZeroDivisionError, e: raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) @@ -165,49 +154,38 @@ def mod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[1] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex remainder")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - - return _t2w(space, mod) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) def divmod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + div, mod = w_complex1.divmod(w_complex2) except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex divmod()")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - w_div = _t2w(space, div) - w_mod = _t2w(space, mod) - return space.newtuple([w_div, w_mod]) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) + return space.newtuple([div, mod]) def floordiv__Complex_Complex(space, w_complex1, w_complex2): + # don't care about the slight slowdown you get from using divmod try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[0] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex floordiv()")) - div = (math.floor(div[0]), 0.0) - return _t2w(space, div) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) -def pow__Complex_Complex_ANY(space, w_complex1, w_complex2, thirdArg): +def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) + int_exponent = int(w_exponent.realval) try: - v = _w2t(space, w_complex1) - exponent = _w2t(space, w_complex2) - int_exponent = int(exponent[0]) - if exponent[1] == 0.0 and exponent[0] == int_exponent: - p = _powi(v, int_exponent) + if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: + w_p = w_complex.pow_int(int_exponent) else: - p = _pow(v, exponent) + w_p = w_complex.pow(w_exponent) except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("0.0 to a negative or complex power")) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("complex exponentiation")) - return _t2w(space, p) + return w_p def neg__Complex(space, w_complex): return W_ComplexObject(-w_complex.realval, -w_complex.imagval) Modified: pypy/trunk/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_complexobject.py Mon Dec 6 21:08:13 2010 @@ -1,5 +1,6 @@ import py -from pypy.objspace.std import complexobject as cobj +from pypy.objspace.std.complexobject import W_ComplexObject, \ + pow__Complex_Complex_ANY from pypy.objspace.std import complextype as cobjtype from pypy.objspace.std.multimethod import FailedToImplement from pypy.objspace.std.stringobject import W_StringObject @@ -11,7 +12,7 @@ def _test_instantiation(self): def _t_complex(r=0.0,i=0.0): - c = cobj.W_ComplexObject(r, i) + c = W_ComplexObject(r, i) assert c.real == float(r) and c.imag == float(i) pairs = ( (1, 1), @@ -38,21 +39,31 @@ test_cparse('.e+5', '.e+5', '0.0') def test_pow(self): - assert cobj._pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) - assert cobj._pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) - rr, ir = cobj._pow((0.0,1.0),(2.0,0.0)) + def _pow((r1, i1), (r2, i2)): + w_res = W_ComplexObject(r1, i1).pow(W_ComplexObject(r2, i2)) + return w_res.realval, w_res.imagval + assert _pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) + assert _pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) + rr, ir = _pow((0.0,1.0),(2.0,0.0)) assert abs(-1.0 - rr) < EPS assert abs(0.0 - ir) < EPS - assert cobj._powu((0.0,2.0),0) == (1.0,0.0) - assert cobj._powu((0.0,0.0),2) == (0.0,0.0) - assert cobj._powu((0.0,1.0),2) == (-1.0,0.0) - assert cobj._powi((0.0,2.0),0) == (1.0,0.0) - assert cobj._powi((0.0,0.0),2) == (0.0,0.0) - assert cobj._powi((0.0,1.0),2) == (-1.0,0.0) - c = cobj.W_ComplexObject(0.0,1.0) - p = cobj.W_ComplexObject(2.0,0.0) - r = cobj.pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) + def _powu((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_positive_int(n) + return w_res.realval, w_res.imagval + assert _powu((0.0,2.0),0) == (1.0,0.0) + assert _powu((0.0,0.0),2) == (0.0,0.0) + assert _powu((0.0,1.0),2) == (-1.0,0.0) + + def _powi((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_int(n) + return w_res.realval, w_res.imagval + assert _powi((0.0,2.0),0) == (1.0,0.0) + assert _powi((0.0,0.0),2) == (0.0,0.0) + assert _powi((0.0,1.0),2) == (-1.0,0.0) + c = W_ComplexObject(0.0,1.0) + p = W_ComplexObject(2.0,0.0) + r = pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) assert r.realval == -1.0 assert r.imagval == 0.0 From agaynor at codespeak.net Mon Dec 6 21:48:38 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 6 Dec 2010 21:48:38 +0100 (CET) Subject: [pypy-svn] r79856 - in pypy/branch/fast-forward/pypy/module/select: . test Message-ID: <20101206204838.B951F282BAD@codespeak.net> Author: agaynor Date: Mon Dec 6 21:48:35 2010 New Revision: 79856 Added: pypy/branch/fast-forward/pypy/module/select/interp_epoll.py pypy/branch/fast-forward/pypy/module/select/test/test_epoll.py Modified: pypy/branch/fast-forward/pypy/module/select/__init__.py Log: Implemented select.epoll. Modified: pypy/branch/fast-forward/pypy/module/select/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/select/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/select/__init__.py Mon Dec 6 21:48:35 2010 @@ -1,17 +1,33 @@ # Package initialisation from pypy.interpreter.mixedmodule import MixedModule + +import select import sys + class Module(MixedModule): appleveldefs = { 'error': 'app_select.error', } interpleveldefs = { - 'poll' : 'interp_select.poll', + 'poll' : 'interp_select.poll', 'select': 'interp_select.select', } + # TODO: this doesn't feel right... + if hasattr(select, "epoll"): + interpleveldefs['epoll'] = 'interp_epoll.W_Epoll' + symbols = [ + "EPOLLIN", "EPOLLOUT", "EPOLLPRI", "EPOLLERR", "EPOLLHUP", + "EPOLLET", "EPOLLONESHOT", "EPOLLRDNORM", "EPOLLRDBAND", + "EPOLLWRNORM", "EPOLLWRBAND", "EPOLLMSG" + ] + for symbol in symbols: + if hasattr(select, symbol): + interpleveldefs[symbol] = "space.wrap(%s)" % getattr(select, symbol) + + def buildloaders(cls): from pypy.rlib import rpoll for name in rpoll.eventnames: @@ -19,4 +35,3 @@ Module.interpleveldefs[name] = "space.wrap(%r)" % value super(Module, cls).buildloaders() buildloaders = classmethod(buildloaders) - Added: pypy/branch/fast-forward/pypy/module/select/interp_epoll.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/module/select/interp_epoll.py Mon Dec 6 21:48:35 2010 @@ -0,0 +1,184 @@ +from __future__ import with_statement + +import errno +import os + +from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.gateway import interp2app, unwrap_spec, ObjSpace, W_Root +from pypy.interpreter.error import OperationError, wrap_oserror, operationerrfmt +from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.module.select.interp_select import as_fd_w +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.tool import rffi_platform +from pypy.rlib._rsocket_rffi import socketclose, FD_SETSIZE +from pypy.rlib.rposix import get_errno +from pypy.translator.tool.cbuild import ExternalCompilationInfo + + +eci = ExternalCompilationInfo( + includes = ['sys/epoll.h'] +) + +class CConfig: + _compilation_info_ = eci + + +CConfig.epoll_data = rffi_platform.Struct("union epoll_data", [ + ("fd", rffi.INT), +]) +CConfig.epoll_event = rffi_platform.Struct("struct epoll_event", [ + ("events", rffi.UINT), + ("data", CConfig.epoll_data) +]) + +for symbol in ["EPOLL_CTL_ADD", "EPOLL_CTL_MOD", "EPOLL_CTL_DEL"]: + setattr(CConfig, symbol, rffi_platform.DefinedConstantInteger(symbol)) + +cconfig = rffi_platform.configure(CConfig) + +epoll_event = cconfig["epoll_event"] +EPOLL_CTL_ADD = cconfig["EPOLL_CTL_ADD"] +EPOLL_CTL_MOD = cconfig["EPOLL_CTL_MOD"] +EPOLL_CTL_DEL = cconfig["EPOLL_CTL_DEL"] + +epoll_create = rffi.llexternal( + "epoll_create", [rffi.INT], rffi.INT, compilation_info=eci +) +epoll_ctl = rffi.llexternal( + "epoll_ctl", + [rffi.INT, rffi.INT, rffi.INT, lltype.Ptr(epoll_event)], + rffi.INT, + compilation_info=eci +) +epoll_wait = rffi.llexternal( + "epoll_wait", + [rffi.INT, lltype.Ptr(rffi.CArray(epoll_event)), rffi.INT, rffi.INT], + rffi.INT, + compilation_info=eci, +) + +def exception_from_errno(space, w_type): + errno = get_errno() + msg = os.strerror(errno) + w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + return OperationError(w_type, w_error) + +class W_Epoll(Wrappable): + def __init__(self, space, epfd): + self.space = space + self.epfd = epfd + + @unwrap_spec(ObjSpace, W_Root, int) + def descr__new__(space, w_subtype, sizehint=-1): + if sizehint == -1: + sizehint = FD_SETSIZE - 1 + elif sizehint < 0: + raise operationerrfmt(space.w_ValueError, + "sizehint must be greater than zero, got %d", sizehint + ) + epfd = epoll_create(sizehint) + if epfd < 0: + raise exception_from_errno(space, space.w_IOError) + + return space.wrap(W_Epoll(space, epfd)) + + @unwrap_spec(ObjSpace, W_Root, int) + def descr_fromfd(space, w_cls, fd): + return space.wrap(W_Epoll(space, fd)) + + def __del__(self): + self.close() + + def check_closed(self): + if self.epfd < 0: + raise OperationError(self.space.w_ValueError, + self.space.wrap("I/O operation on closed epoll fd") + ) + + def close(self): + if not self.epfd < 0: + socketclose(self.epfd) + self.epfd = -1 + + def epoll_ctl(self, ctl, w_fd, eventmask, ignore_ebadf=False): + fd = as_fd_w(self.space, w_fd) + with lltype.scoped_alloc(epoll_event) as ev: + ev.c_events = rffi.cast(rffi.UINT, eventmask) + ev.c_data.c_fd = fd + + result = epoll_ctl(self.epfd, ctl, fd, ev) + if ignore_ebadf and get_errno() == errno.EBADF: + result = 0 + if result < 0: + raise exception_from_errno(self.space, self.space.w_IOError) + + def descr_get_closed(space, self): + return space.wrap(self.epfd < 0) + + @unwrap_spec("self", ObjSpace) + def descr_fileno(self, space): + self.check_closed() + return space.wrap(self.epfd) + + @unwrap_spec("self", ObjSpace) + def descr_close(self, space): + self.check_closed() + self.close() + + @unwrap_spec("self", ObjSpace, W_Root, int) + def descr_register(self, space, w_fd, eventmask=-1): + self.check_closed() + self.epoll_ctl(EPOLL_CTL_ADD, w_fd, eventmask) + + @unwrap_spec("self", ObjSpace, W_Root) + def descr_unregister(self, space, w_fd): + self.check_closed() + self.epoll_ctl(EPOLL_CTL_DEL, w_fd, 0, ignore_ebadf=True) + + @unwrap_spec("self", ObjSpace, W_Root, int) + def descr_modify(self, space, w_fd, eventmask=-1): + self.check_closed() + self.epoll_ctl(EPOLL_CTL_MOD, w_fd, eventmask) + + @unwrap_spec("self", ObjSpace, float, int) + def descr_poll(self, space, timeout=-1.0, maxevents=-1): + self.check_closed() + if timeout < 0: + timeout = -1.0 + else: + timeout *= 1000.0 + + if maxevents == -1: + maxevents = FD_SETSIZE - 1 + elif maxevents < 1: + raise operationerrfmt(space.w_ValueError, + "maxevents must be greater than 0, not %d", maxevents + ) + + with lltype.scoped_alloc(rffi.CArray(epoll_event), maxevents) as evs: + nfds = epoll_wait(self.epfd, evs, maxevents, int(timeout)) + if nfds < 0: + raise exception_from_errno(space, space.w_IOError) + + elist_w = [None] * nfds + for i in xrange(nfds): + event = evs[i] + elist_w[i] = space.newtuple( + [space.wrap(event.c_data.c_fd), space.wrap(event.c_events)] + ) + return space.newlist(elist_w) + + +W_Epoll.typedef = TypeDef("select.epoll", + __new__ = interp2app(W_Epoll.descr__new__.im_func), + fromfd = interp2app(W_Epoll.descr_fromfd.im_func, as_classmethod=True), + + closed = GetSetProperty(W_Epoll.descr_get_closed), + fileno = interp2app(W_Epoll.descr_fileno), + close = interp2app(W_Epoll.descr_close), + register = interp2app(W_Epoll.descr_register), + unregister = interp2app(W_Epoll.descr_unregister), + modify = interp2app(W_Epoll.descr_modify), + poll = interp2app(W_Epoll.descr_poll), +) +W_Epoll.typedef.acceptable_as_base_class = False Added: pypy/branch/fast-forward/pypy/module/select/test/test_epoll.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/module/select/test/test_epoll.py Mon Dec 6 21:48:35 2010 @@ -0,0 +1,230 @@ +import py + +from pypy.conftest import gettestobjspace + + +class AppTestEpoll(object): + def setup_class(cls): + cls.space = gettestobjspace(usemodules=["select", "_socket", "posix"]) + + import errno + import select + + if not hasattr(select, "epoll"): + py.test.skip("test requires linux 2.6") + try: + select.epoll() + except IOError, e: + if e.errno == errno.ENOSYS: + py.test.skip("kernel doesn't support epoll()") + + def setup_method(self, meth): + self.w_sockets = self.space.wrap([]) + + def teardown_method(self, meth): + for socket in self.space.unpackiterable(self.w_sockets): + self.space.call_method(socket, "close") + + def test_create(self): + import select + + ep = select.epoll(16) + assert ep.fileno() > 0 + assert not ep.closed + ep.close() + assert ep.closed + raises(ValueError, ep.fileno) + + def test_badcreate(self): + import select + + raises(TypeError, select.epoll, 1, 2, 3) + raises(TypeError, select.epoll, 'foo') + raises(TypeError, select.epoll, None) + raises(TypeError, select.epoll, ()) + raises(TypeError, select.epoll, ['foo']) + raises(TypeError, select.epoll, {}) + + def test_add(self): + import posix + import select + import socket + + server_socket = socket.socket() + server_socket.bind(('127.0.0.1', 0)) + server_socket.listen(1) + client = socket.socket() + client.setblocking(False) + raises(socket.error, + client.connect, ('127.0.0.1', server_socket.getsockname()[1]) + ) + server, addr = server_socket.accept() + + self.sockets.extend([server_socket, client, server]) + + ep = select.epoll(2) + ep.register(server, select.EPOLLIN | select.EPOLLOUT) + ep.register(client, select.EPOLLIN | select.EPOLLOUT) + ep.close() + + # adding by object w/ fileno works, too. + ep = select.epoll(2) + ep.register(server.fileno(), select.EPOLLIN | select.EPOLLOUT) + ep.register(client.fileno(), select.EPOLLIN | select.EPOLLOUT) + ep.close() + + ep = select.epoll(2) + # TypeError: argument must be an int, or have a fileno() method. + raises(TypeError, ep.register, object(), select.EPOLLIN | select.EPOLLOUT) + raises(TypeError, ep.register, None, select.EPOLLIN | select.EPOLLOUT) + # ValueError: file descriptor cannot be a negative integer (-1) + raises(ValueError, ep.register, -1, select.EPOLLIN | select.EPOLLOUT) + # IOError: [Errno 9] Bad file descriptor + raises(IOError, ep.register, 10000, select.EPOLLIN | select.EPOLLOUT) + # registering twice also raises an exception + ep.register(server, select.EPOLLIN | select.EPOLLOUT) + raises(IOError, ep.register, server, select.EPOLLIN | select.EPOLLOUT) + ep.close() + + def test_fromfd(self): + import errno + import select + import socket + + server_socket = socket.socket() + server_socket.bind(('127.0.0.1', 0)) + server_socket.listen(1) + client = socket.socket() + client.setblocking(False) + raises(socket.error, + client.connect, ('127.0.0.1', server_socket.getsockname()[1]) + ) + server, addr = server_socket.accept() + + self.sockets.extend([server_socket, client, server]) + + + ep1 = select.epoll(2) + ep2 = select.epoll.fromfd(ep1.fileno()) + + ep2.register(server.fileno(), select.EPOLLIN | select.EPOLLOUT) + ep2.register(client.fileno(), select.EPOLLIN | select.EPOLLOUT) + + events1 = ep1.poll(1, 4) + events2 = ep2.poll(0.9, 4) + assert len(events1) == 2 + assert len(events2) == 2 + ep1.close() + + exc_info = raises(IOError, ep2.poll, 1, 4) + assert exc_info.value.args[0] == errno.EBADF + + def test_control_and_wait(self): + import select + import socket + import time + + server_socket = socket.socket() + server_socket.bind(('127.0.0.1', 0)) + server_socket.listen(1) + client = socket.socket() + client.setblocking(False) + raises(socket.error, + client.connect, ('127.0.0.1', server_socket.getsockname()[1]) + ) + server, addr = server_socket.accept() + + self.sockets.extend([server_socket, client, server]) + + + ep = select.epoll(16) + ep.register(server.fileno(), + select.EPOLLIN | select.EPOLLOUT | select.EPOLLET + ) + ep.register(client.fileno(), + select.EPOLLIN | select.EPOLLOUT | select.EPOLLET + ) + + now = time.time() + events = ep.poll(1, 4) + then = time.time() + assert then - now < 0.1 + + events.sort() + expected = [ + (client.fileno(), select.EPOLLOUT), + (server.fileno(), select.EPOLLOUT) + ] + expected.sort() + + assert events == expected + assert then - now < 0.01 + + now = time.time() + events = ep.poll(timeout=2.1, maxevents=4) + then = time.time() + assert not events + + client.send("Hello!") + server.send("world!!!") + + now = time.time() + events = ep.poll(1, 4) + then = time.time() + assert then - now < 0.01 + + events.sort() + expected = [ + (client.fileno(), select.EPOLLIN | select.EPOLLOUT), + (server.fileno(), select.EPOLLIN | select.EPOLLOUT) + ] + expected.sort() + + assert events == expected + + ep.unregister(client.fileno()) + ep.modify(server.fileno(), select.EPOLLOUT) + + now = time.time() + events = ep.poll(1, 4) + then = time.time() + assert then - now < 0.01 + + expected = [(server.fileno(), select.EPOLLOUT)] + assert events == expected + + def test_errors(self): + import select + + raises(ValueError, select.epoll, -2) + raises(ValueError, select.epoll().register, -1, select.EPOLLIN) + + def test_unregister_closed(self): + import select + import socket + import time + + server_socket = socket.socket() + server_socket.bind(('127.0.0.1', 0)) + server_socket.listen(1) + client = socket.socket() + client.setblocking(False) + raises(socket.error, + client.connect, ('127.0.0.1', server_socket.getsockname()[1]) + ) + server, addr = server_socket.accept() + + self.sockets.extend([server_socket, client, server]) + + + fd = server.fileno() + ep = select.epoll(16) + ep.register(server) + + now = time.time() + events = ep.poll(1, 4) + then = time.time() + assert then - now < 0.01 + + server.close() + ep.unregister(fd) From afa at codespeak.net Mon Dec 6 23:56:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 6 Dec 2010 23:56:43 +0100 (CET) Subject: [pypy-svn] r79857 - in pypy/branch/fast-forward/pypy/module/__builtin__: . test Message-ID: <20101206225643.B0B9A282B90@codespeak.net> Author: afa Date: Mon Dec 6 23:56:39 2010 New Revision: 79857 Modified: pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py pypy/branch/fast-forward/pypy/module/__builtin__/test/test_buffer.py Log: Implement memoryview.suboffsets Modified: pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py (original) +++ pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py Mon Dec 6 23:56:39 2010 @@ -113,6 +113,14 @@ return space.newtuple([space.wrap(self.getlength())]) def w_get_strides(space, self): return space.newtuple([space.wrap(1)]) + def w_get_suboffsets(space, self): + if isinstance(self.buf, buffer.SubBuffer): + offset = self.buf.offset + elif isinstance(self.buf, buffer.RWSubBuffer): + offset = self.buf.offset + else: + offset = 0 + return space.newtuple([space.wrap(offset)]) @unwrap_spec(ObjSpace, W_Root, W_Root) @@ -144,6 +152,6 @@ readonly = GetSetProperty(W_MemoryView.w_is_readonly), shape = GetSetProperty(W_MemoryView.w_get_shape), strides = GetSetProperty(W_MemoryView.w_get_strides), - #suboffsets? undocumented + suboffsets = GetSetProperty(W_MemoryView.w_get_suboffsets), ) W_MemoryView.typedef.acceptable_as_base_class = False Modified: pypy/branch/fast-forward/pypy/module/__builtin__/test/test_buffer.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__builtin__/test/test_buffer.py (original) +++ pypy/branch/fast-forward/pypy/module/__builtin__/test/test_buffer.py Mon Dec 6 23:56:39 2010 @@ -192,3 +192,10 @@ assert v.shape == (100,) assert v.ndim == 1 assert v.strides == (1,) + + def test_suboffsets(self): + v = memoryview("a"*100) + assert v.suboffsets == (0,) + v = memoryview(buffer("a"*100, 2)) + assert v.shape == (98,) + assert v.suboffsets == (2,) From afa at codespeak.net Tue Dec 7 00:00:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 00:00:11 +0100 (CET) Subject: [pypy-svn] r79858 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101206230011.9A547282B90@codespeak.net> Author: afa Date: Tue Dec 7 00:00:10 2010 New Revision: 79858 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_code.py Log: Call gc_collect() to clean a weak reference Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_code.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_code.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_code.py Tue Dec 7 00:00:10 2010 @@ -134,6 +134,7 @@ coderef = weakref.ref(f.__code__, callback) self.assertTrue(bool(coderef())) del f + test_support.collect() self.assertFalse(bool(coderef())) self.assertTrue(self.called) From afa at codespeak.net Tue Dec 7 00:24:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 00:24:51 +0100 (CET) Subject: [pypy-svn] r79859 - in pypy/branch/fast-forward: lib-python/modified-2.7.0/test pypy/objspace/std Message-ID: <20101206232451.1CEB3282B90@codespeak.net> Author: afa Date: Tue Dec 7 00:24:48 2010 New Revision: 79859 Added: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_unicode.py - copied, changed from r79824, pypy/branch/fast-forward/lib-python/2.7.0/test/test_unicode.py Modified: pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py pypy/branch/fast-forward/pypy/objspace/std/unicodetype.py Log: Give argument names to unicode.encode(), allow test_expandtabs to raise a MemoryError Copied: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_unicode.py (from r79824, pypy/branch/fast-forward/lib-python/2.7.0/test/test_unicode.py) ============================================================================== --- pypy/branch/fast-forward/lib-python/2.7.0/test/test_unicode.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_unicode.py Tue Dec 7 00:24:48 2010 @@ -1045,7 +1045,8 @@ # to take a 64-bit long, this test should apply to all platforms. if sys.maxint > (1 << 32) or struct.calcsize('P') != 4: return - self.assertRaises(OverflowError, u't\tt\t'.expandtabs, sys.maxint) + self.assertRaises((OverflowError, MemoryError), + u't\tt\t'.expandtabs, sys.maxint) def test__format__(self): def test(value, format, expected): Modified: pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/unicodeobject.py Tue Dec 7 00:24:48 2010 @@ -838,11 +838,11 @@ parts = _split_with(self, u'\t') result = [parts[0]] prevsize = 0 - totalsize = 0 for ch in parts[0]: prevsize += 1 if ch == u"\n" or ch == u"\r": prevsize = 0 + totalsize = prevsize for i in range(1, len(parts)): pad = tabsize - prevsize % tabsize Modified: pypy/branch/fast-forward/pypy/objspace/std/unicodetype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/unicodetype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/unicodetype.py Tue Dec 7 00:24:48 2010 @@ -48,6 +48,7 @@ ' arguments start and end are\ninterpreted as in' ' slice notation.') unicode_encode = SMM('encode', 3, defaults=(None, None), + argnames=['encoding', 'errors'], doc='S.encode([encoding[,errors]]) -> string or' ' unicode\n\nEncodes S using the codec registered' ' for encoding. encoding defaults\nto the default' From afa at codespeak.net Tue Dec 7 00:25:55 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 00:25:55 +0100 (CET) Subject: [pypy-svn] r79860 - pypy/branch/fast-forward/pypy/module/cpyext/include Message-ID: <20101206232555.59EF6282BD4@codespeak.net> Author: afa Date: Tue Dec 7 00:25:53 2010 New Revision: 79860 Modified: pypy/branch/fast-forward/pypy/module/cpyext/include/pythonrun.h Log: Add Py_Py3kWarningFlag, it will probably stay to zero Modified: pypy/branch/fast-forward/pypy/module/cpyext/include/pythonrun.h ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/include/pythonrun.h (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/include/pythonrun.h Tue Dec 7 00:25:53 2010 @@ -8,6 +8,9 @@ void Py_FatalError(const char *msg); +/* the -3 option will probably not be implemented */ +#define Py_Py3kWarningFlag 0 + #ifdef __cplusplus } #endif From afa at codespeak.net Tue Dec 7 00:34:53 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 00:34:53 +0100 (CET) Subject: [pypy-svn] r79861 - pypy/branch/fast-forward/pypy/module/signal Message-ID: <20101206233453.D8589282BDC@codespeak.net> Author: afa Date: Tue Dec 7 00:34:52 2010 New Revision: 79861 Modified: pypy/branch/fast-forward/pypy/module/signal/__init__.py pypy/branch/fast-forward/pypy/module/signal/interp_signal.py Log: Implement signal.siginterrupt() Modified: pypy/branch/fast-forward/pypy/module/signal/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/signal/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/signal/__init__.py Tue Dec 7 00:34:52 2010 @@ -15,6 +15,7 @@ if os.name == 'posix': interpleveldefs['alarm'] = 'interp_signal.alarm' interpleveldefs['pause'] = 'interp_signal.pause' + interpleveldefs['siginterrupt'] = 'interp_signal.siginterrupt' appleveldefs = { 'default_int_handler': 'app_signal.default_int_handler', Modified: pypy/branch/fast-forward/pypy/module/signal/interp_signal.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/signal/interp_signal.py (original) +++ pypy/branch/fast-forward/pypy/module/signal/interp_signal.py Tue Dec 7 00:34:52 2010 @@ -7,7 +7,7 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo import py from pypy.tool import autopath -from pypy.rlib import jit +from pypy.rlib import jit, rposix from pypy.rlib.rarithmetic import intmask def setup(): @@ -53,6 +53,7 @@ pure_function=True) c_alarm = external('alarm', [rffi.INT], rffi.INT) c_pause = external('pause', [], rffi.INT) +c_siginterrupt = external('siginterrupt', [rffi.INT, rffi.INT], rffi.INT) class SignalActionFlag(AbstractActionFlag): @@ -244,3 +245,10 @@ old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) set_wakeup_fd.unwrap_spec = [ObjSpace, int] + +def siginterrupt(space, signum, flag): + check_signum(space, signum) + if rffi.cast(lltype.Signed, c_siginterrupt(signum, flag)) < 0: + errno = rposix.get_errno() + raise OperationError(space.w_RuntimeError, space.wrap(errno)) +siginterrupt.unwrap_spec = [ObjSpace, int, int] From afa at codespeak.net Tue Dec 7 00:35:42 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 00:35:42 +0100 (CET) Subject: [pypy-svn] r79862 - pypy/branch/fast-forward/pypy/module/signal/test Message-ID: <20101206233542.B0B2F282BDD@codespeak.net> Author: afa Date: Tue Dec 7 00:35:41 2010 New Revision: 79862 Modified: pypy/branch/fast-forward/pypy/module/signal/test/test_signal.py Log: Test for the previous commit Modified: pypy/branch/fast-forward/pypy/module/signal/test/test_signal.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/signal/test/test_signal.py (original) +++ pypy/branch/fast-forward/pypy/module/signal/test/test_signal.py Tue Dec 7 00:35:41 2010 @@ -196,6 +196,14 @@ # signal.signal(signal.SIGUSR1, signal.SIG_DFL) + def test_siginterrupt(self): + import signal + signum = signal.SIGUSR1 + oldhandler = signal.signal(signum, lambda x,y: None) + try: + signal.siginterrupt(signum, 0) + finally: + signal.signal(signum, oldhandler) class AppTestSignalSocket: From agaynor at codespeak.net Tue Dec 7 00:43:34 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 7 Dec 2010 00:43:34 +0100 (CET) Subject: [pypy-svn] r79863 - in pypy/branch/fast-forward/pypy/module/select: . test Message-ID: <20101206234334.7FA30282BDC@codespeak.net> Author: agaynor Date: Tue Dec 7 00:43:33 2010 New Revision: 79863 Modified: pypy/branch/fast-forward/pypy/module/select/interp_epoll.py (props changed) pypy/branch/fast-forward/pypy/module/select/test/test_epoll.py (props changed) Log: Fixed EOL. From afa at codespeak.net Tue Dec 7 01:30:34 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 01:30:34 +0100 (CET) Subject: [pypy-svn] r79864 - pypy/branch/fast-forward/pypy/module/__builtin__ Message-ID: <20101207003034.27CEB282B90@codespeak.net> Author: afa Date: Tue Dec 7 01:30:32 2010 New Revision: 79864 Modified: pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py Log: Try to fix translation Modified: pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py (original) +++ pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py Tue Dec 7 01:30:32 2010 @@ -114,10 +114,11 @@ def w_get_strides(space, self): return space.newtuple([space.wrap(1)]) def w_get_suboffsets(space, self): - if isinstance(self.buf, buffer.SubBuffer): - offset = self.buf.offset - elif isinstance(self.buf, buffer.RWSubBuffer): - offset = self.buf.offset + buf = self.buf + if isinstance(buf, buffer.SubBuffer): + offset = buf.offset + elif isinstance(buf, buffer.RWSubBuffer): + offset = buf.offset else: offset = 0 return space.newtuple([space.wrap(offset)]) From afa at codespeak.net Tue Dec 7 02:23:36 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 02:23:36 +0100 (CET) Subject: [pypy-svn] r79865 - pypy/branch/fast-forward/pypy/module/select Message-ID: <20101207012336.23FAF282B90@codespeak.net> Author: afa Date: Tue Dec 7 02:23:33 2010 New Revision: 79865 Modified: pypy/branch/fast-forward/pypy/module/select/interp_epoll.py Log: Maybe fix translation Modified: pypy/branch/fast-forward/pypy/module/select/interp_epoll.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/select/interp_epoll.py (original) +++ pypy/branch/fast-forward/pypy/module/select/interp_epoll.py Tue Dec 7 02:23:33 2010 @@ -104,7 +104,7 @@ fd = as_fd_w(self.space, w_fd) with lltype.scoped_alloc(epoll_event) as ev: ev.c_events = rffi.cast(rffi.UINT, eventmask) - ev.c_data.c_fd = fd + rffi.setintfield(ev.c_data, 'c_fd', fd) result = epoll_ctl(self.epfd, ctl, fd, ev) if ignore_ebadf and get_errno() == errno.EBADF: From arigo at codespeak.net Tue Dec 7 15:21:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Dec 2010 15:21:37 +0100 (CET) Subject: [pypy-svn] r79866 - in pypy/trunk/pypy/interpreter/pyparser: . test Message-ID: <20101207142137.57B215080B@codespeak.net> Author: arigo Date: Tue Dec 7 15:21:35 2010 New Revision: 79866 Modified: pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py Log: issue563 testing On multi-line EOF SyntaxErrors, report the start line instead of the EOF line. Modified: pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py Tue Dec 7 15:21:35 2010 @@ -78,6 +78,7 @@ contline = None indents = [0] last_comment = '' + parenlevstart = (0, 0, "") # make the annotator happy endDFA = automata.DFA([], []) @@ -85,7 +86,7 @@ line = '' pos = 0 lines.append("") - strstart = (0, 0) + strstart = (0, 0, "") for line in lines: lnum = lnum + 1 pos, max = 0, len(line) @@ -93,7 +94,8 @@ if contstr: if not line: raise TokenError("EOF while scanning triple-quoted string", - line, lnum-1, 0, token_list) + strstart[2], strstart[0], strstart[1], + token_list) endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch @@ -146,8 +148,11 @@ else: # continued statement if not line: + start = 0 + if parenlev > 0: + lnum, start, line = parenlevstart raise TokenError("EOF in multi-line statement", line, - lnum, 0, token_list) + lnum, start, token_list) continued = 0 while pos < max: @@ -187,7 +192,7 @@ token_list.append(tok) last_comment = '' else: - strstart = (lnum, start) + strstart = (lnum, start, line) contstr = line[start:] contline = line break @@ -195,7 +200,7 @@ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string - strstart = (lnum, start) + strstart = (lnum, start, line) endDFA = (endDFAs[initial] or endDFAs[token[1]] or endDFAs[token[2]]) contstr, needcont = line[start:], 1 @@ -212,6 +217,8 @@ continued = 1 else: if initial in '([{': + if parenlev == 0: + parenlevstart = (lnum, start, line) parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py Tue Dec 7 15:21:35 2010 @@ -64,12 +64,20 @@ assert exc.lineno == 1 assert exc.offset == 5 assert exc.text.startswith("name another for") - exc = py.test.raises(SyntaxError, parse, "\"blah").value + exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value assert exc.msg == "EOL while scanning single-quoted string" - exc = py.test.raises(SyntaxError, parse, "'''\n").value + assert exc.lineno == 1 + assert exc.offset == 4 + exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value assert exc.msg == "EOF while scanning triple-quoted string" + assert exc.lineno == 1 + assert exc.offset == 4 for input in ("())", "(()", "((", "))"): py.test.raises(SyntaxError, parse, input) + exc = py.test.raises(SyntaxError, parse, "x = (\n\n(),\n(),").value + assert exc.msg == "EOF in multi-line statement" + assert exc.lineno == 1 + assert exc.offset == 4 def test_is(self): self.parse("x is y") From afa at codespeak.net Tue Dec 7 16:27:38 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 16:27:38 +0100 (CET) Subject: [pypy-svn] r79867 - pypy/branch/fast-forward/pypy/module/_multiprocessing Message-ID: <20101207152738.9DA57282BAD@codespeak.net> Author: afa Date: Tue Dec 7 16:27:35 2010 New Revision: 79867 Modified: pypy/branch/fast-forward/pypy/module/_multiprocessing/interp_semaphore.py Log: Mac OS X does not need (and does not have) the librt library. Modified: pypy/branch/fast-forward/pypy/module/_multiprocessing/interp_semaphore.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_multiprocessing/interp_semaphore.py (original) +++ pypy/branch/fast-forward/pypy/module/_multiprocessing/interp_semaphore.py Tue Dec 7 16:27:35 2010 @@ -30,11 +30,16 @@ else: from pypy.rlib import rposix + if sys.platform == 'darwin': + libraries = [] + else: + libraries = ['rt'] + eci = ExternalCompilationInfo( includes = ['sys/time.h', 'limits.h', 'semaphore.h'], - libraries = ['rt'], + libraries = libraries, ) class CConfig: From arigo at codespeak.net Tue Dec 7 17:16:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Dec 2010 17:16:13 +0100 (CET) Subject: [pypy-svn] r79868 - in pypy/trunk/pypy: interpreter module/imp Message-ID: <20101207161613.EB2A1282BAD@codespeak.net> Author: arigo Date: Tue Dec 7 17:16:11 2010 New Revision: 79868 Modified: pypy/trunk/pypy/interpreter/mixedmodule.py pypy/trunk/pypy/module/imp/importing.py Log: Fix issue557 by tweaking what occurs when reload()ing a built-in module. Now the state that is saved and restored should be the state after the module is first imported *and* after startup() has been called. Modified: pypy/trunk/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/trunk/pypy/interpreter/mixedmodule.py (original) +++ pypy/trunk/pypy/interpreter/mixedmodule.py Tue Dec 7 17:16:11 2010 @@ -13,6 +13,10 @@ applevel_name = None expose__file__attribute = True + + # The following attribute is None as long as the module has not been + # imported yet, and when it has been, it is mod.__dict__.items() just + # after startup(). w_initialdict = None def __init__(self, space, w_name): @@ -26,8 +30,14 @@ """This is called each time the module is imported or reloaded """ if self.w_initialdict is not None: + # the module was already imported. Refresh its content with + # the saved dict, as done with built-in and extension modules + # on CPython. space.call_method(self.w_dict, 'update', self.w_initialdict) - Module.init(self, space) + else: + Module.init(self, space) + if not self.lazy and self.w_initialdict is None: + self.w_initialdict = space.call_method(self.w_dict, 'items') def get_applevel_name(cls): """ NOT_RPYTHON """ @@ -96,6 +106,7 @@ def _freeze_(self): self.getdict() + self.w_initialdict = None self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant Modified: pypy/trunk/pypy/module/imp/importing.py ============================================================================== --- pypy/trunk/pypy/module/imp/importing.py (original) +++ pypy/trunk/pypy/module/imp/importing.py Tue Dec 7 17:16:11 2010 @@ -529,7 +529,7 @@ space.sys.setmodule(w_module) raise finally: - space.reloading_modules.clear() + del space.reloading_modules[modulename] # __________________________________________________________________ From arigo at codespeak.net Tue Dec 7 17:18:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Dec 2010 17:18:48 +0100 (CET) Subject: [pypy-svn] r79869 - in pypy/trunk/pypy: config jit/backend/llsupport jit/metainterp jit/metainterp/optimizeopt jit/metainterp/test Message-ID: <20101207161848.9BE16282BAD@codespeak.net> Author: arigo Date: Tue Dec 7 17:18:46 2010 New Revision: 79869 Modified: pypy/trunk/pypy/config/pypyoption.py pypy/trunk/pypy/config/translationoption.py pypy/trunk/pypy/jit/backend/llsupport/llmodel.py pypy/trunk/pypy/jit/metainterp/optimizeopt/__init__.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/jit/metainterp/test/test_compile.py pypy/trunk/pypy/jit/metainterp/test/test_fficall.py pypy/trunk/pypy/jit/metainterp/test/test_optimizefficall.py pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py pypy/trunk/pypy/jit/metainterp/warmspot.py Log: Add a config option "jit_ffi", which is True only if we use the pypy module _ffi (and in some tests). Enable or disable OptFfiCall based on that. Modified: pypy/trunk/pypy/config/pypyoption.py ============================================================================== --- pypy/trunk/pypy/config/pypyoption.py (original) +++ pypy/trunk/pypy/config/pypyoption.py Tue Dec 7 17:18:46 2010 @@ -76,6 +76,7 @@ "_rawffi": [("objspace.usemodules.struct", True)], "cpyext": [("translation.secondaryentrypoints", "cpyext"), ("translation.shared", sys.platform == "win32")], + "_ffi": [("translation.jit_ffi", True)], } module_import_dependencies = { Modified: pypy/trunk/pypy/config/translationoption.py ============================================================================== --- pypy/trunk/pypy/config/translationoption.py (original) +++ pypy/trunk/pypy/config/translationoption.py Tue Dec 7 17:18:46 2010 @@ -117,6 +117,7 @@ ChoiceOption("jit_profiler", "integrate profiler support into the JIT", ["off", "oprofile"], default="off"), + BoolOption("jit_ffi", "optimize libffi calls", default=False), # misc BoolOption("verbose", "Print extra information", default=False), Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Tue Dec 7 17:18:46 2010 @@ -17,7 +17,6 @@ from pypy.jit.backend.llsupport.descr import get_call_descr from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr -from pypy.jit.backend.llsupport.ffisupport import get_call_descr_dynamic from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -247,7 +246,9 @@ return get_call_descr(self.gc_ll_descr, ARGS, RESULT, extrainfo) def calldescrof_dynamic(self, ffi_args, ffi_result, extrainfo=None): - return get_call_descr_dynamic(ffi_args, ffi_result, extrainfo) + from pypy.jit.backend.llsupport import ffisupport + return ffisupport.get_call_descr_dynamic(ffi_args, ffi_result, + extrainfo) def get_overflow_error(self): ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable) Modified: pypy/trunk/pypy/jit/metainterp/optimizeopt/__init__.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/optimizeopt/__init__.py (original) +++ pypy/trunk/pypy/jit/metainterp/optimizeopt/__init__.py Tue Dec 7 17:18:46 2010 @@ -3,7 +3,6 @@ from pypy.jit.metainterp.optimizeopt.intbounds import OptIntBounds from pypy.jit.metainterp.optimizeopt.virtualize import OptVirtualize from pypy.jit.metainterp.optimizeopt.heap import OptHeap -from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall from pypy.jit.metainterp.optimizeopt.string import OptString def optimize_loop_1(metainterp_sd, loop, virtuals=True): @@ -17,6 +16,10 @@ OptVirtualize(), OptString(), OptHeap(), + ] + if metainterp_sd.jit_ffi: + from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall + optimizations = optimizations + [ OptFfiCall(), ] optimizer = Optimizer(metainterp_sd, loop, optimizations, virtuals) Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Tue Dec 7 17:18:46 2010 @@ -1216,7 +1216,8 @@ logger_ops = None def __init__(self, cpu, options, - ProfilerClass=EmptyProfiler, warmrunnerdesc=None): + ProfilerClass=EmptyProfiler, warmrunnerdesc=None, + jit_ffi=True): self.cpu = cpu self.stats = self.cpu.stats self.options = options @@ -1226,6 +1227,7 @@ self.profiler = ProfilerClass() self.profiler.cpu = cpu self.warmrunnerdesc = warmrunnerdesc + self.jit_ffi = jit_ffi backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] Modified: pypy/trunk/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_compile.py Tue Dec 7 17:18:46 2010 @@ -54,6 +54,7 @@ stats = Stats() profiler = jitprof.EmptyProfiler() warmrunnerdesc = None + jit_ffi = False def log(self, msg, event_kind=None): pass Modified: pypy/trunk/pypy/jit/metainterp/test/test_fficall.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_fficall.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_fficall.py Tue Dec 7 17:18:46 2010 @@ -40,6 +40,6 @@ n += 1 return res # - res = self.meta_interp(f, [0]) + res = self.meta_interp(f, [0], jit_ffi=True) return res Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizefficall.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizefficall.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizefficall.py Tue Dec 7 17:18:46 2010 @@ -31,6 +31,7 @@ class TestFfiCall(BaseTestOptimizeOpt, LLtypeMixin): + jit_ffi = True class namespace: cpu = LLtypeMixin.cpu Modified: pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/trunk/pypy/jit/metainterp/test/test_optimizeopt.py Tue Dec 7 17:18:46 2010 @@ -28,11 +28,12 @@ class FakeMetaInterpStaticData(object): - def __init__(self, cpu): + def __init__(self, cpu, jit_ffi=False): self.cpu = cpu self.profiler = EmptyProfiler() self.options = Fake() self.globaldata = Fake() + self.jit_ffi = jit_ffi def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr @@ -227,6 +228,7 @@ return sorted(boxes, key=lambda box: _kind2count[box.type]) class BaseTestOptimizeOpt(BaseTest): + jit_ffi = False def invent_fail_descr(self, fail_args): if fail_args is None: @@ -261,7 +263,7 @@ loop.token.specnodes = self.unpack_specnodes(spectext) # self.loop = loop - metainterp_sd = FakeMetaInterpStaticData(self.cpu) + metainterp_sd = FakeMetaInterpStaticData(self.cpu, self.jit_ffi) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo if hasattr(self, 'callinfocollection'): Modified: pypy/trunk/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/warmspot.py (original) +++ pypy/trunk/pypy/jit/metainterp/warmspot.py Tue Dec 7 17:18:46 2010 @@ -144,7 +144,8 @@ class WarmRunnerDesc(object): def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, - optimizer=None, ProfilerClass=EmptyProfiler, **kwds): + optimizer=None, ProfilerClass=EmptyProfiler, + jit_ffi=None, **kwds): pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) self.memory_manager = memmgr.MemoryManager() @@ -162,7 +163,7 @@ elif self.opt.listops: self.prejit_optimizations_minimal_inline(policy, graphs) - self.build_meta_interp(ProfilerClass) + self.build_meta_interp(ProfilerClass, jit_ffi) self.make_args_specifications() # from pypy.jit.metainterp.virtualref import VirtualRefInfo @@ -280,11 +281,14 @@ translate_support_code, gcdescr=self.gcdescr) self.cpu = cpu - def build_meta_interp(self, ProfilerClass): + def build_meta_interp(self, ProfilerClass, jit_ffi=None): + if jit_ffi is None: + jit_ffi = self.translator.config.translation.jit_ffi self.metainterp_sd = MetaInterpStaticData(self.cpu, self.opt, ProfilerClass=ProfilerClass, - warmrunnerdesc=self) + warmrunnerdesc=self, + jit_ffi=jit_ffi) def make_virtualizable_infos(self): vinfos = {} From arigo at codespeak.net Tue Dec 7 17:23:42 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Dec 2010 17:23:42 +0100 (CET) Subject: [pypy-svn] r79870 - in pypy/trunk/pypy/interpreter/pyparser: . test Message-ID: <20101207162342.D3862282BAD@codespeak.net> Author: arigo Date: Tue Dec 7 17:23:41 2010 New Revision: 79870 Modified: pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py Log: In the report of these SyntaxErrors, shift the caret one place to the right in order to point exactly to the start of the string or the parenthesis that is never closed. Modified: pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py Tue Dec 7 17:23:41 2010 @@ -94,7 +94,7 @@ if contstr: if not line: raise TokenError("EOF while scanning triple-quoted string", - strstart[2], strstart[0], strstart[1], + strstart[2], strstart[0], strstart[1]+1, token_list) endmatch = endDFA.recognize(line) if endmatch >= 0: @@ -151,6 +151,7 @@ start = 0 if parenlev > 0: lnum, start, line = parenlevstart + start += 1 raise TokenError("EOF in multi-line statement", line, lnum, start, token_list) continued = 0 @@ -237,7 +238,7 @@ start = pos if start Author: arigo Date: Tue Dec 7 17:42:24 2010 New Revision: 79871 Modified: pypy/trunk/pypy/module/_lsprof/test/test_cprofile.py Log: Add a test for issue423. Modified: pypy/trunk/pypy/module/_lsprof/test/test_cprofile.py ============================================================================== --- pypy/trunk/pypy/module/_lsprof/test/test_cprofile.py (original) +++ pypy/trunk/pypy/module/_lsprof/test/test_cprofile.py Tue Dec 7 17:42:24 2010 @@ -2,9 +2,10 @@ from pypy.conftest import gettestobjspace, option class AppTestCProfile(object): + keywords = {} def setup_class(cls): - space = gettestobjspace(usemodules=('_lsprof',)) + space = gettestobjspace(usemodules=('_lsprof',), **cls.keywords) cls.w_expected_output = space.wrap(expected_output) cls.space = space cls.w_file = space.wrap(__file__) @@ -148,6 +149,12 @@ finally: sys.path.pop(0) + +class AppTestDifferentBytecode(AppTestCProfile): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True, + 'objspace.opcodes.CALL_METHOD': True} + + expected_output = {} expected_output['print_stats'] = """\ 126 function calls (106 primitive calls) in 1.000 CPU seconds @@ -165,11 +172,11 @@ 2 0.000 0.000 0.140 0.070 profilee.py:84(helper2_indirect) 8 0.312 0.039 0.400 0.050 profilee.py:88(helper2) 8 0.064 0.008 0.080 0.010 profilee.py:98(subhelper) - 4 0.000 0.000 0.000 0.000 {.*append.*} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} 1 0.000 0.000 0.000 0.000 {.*disable.*} - 12 0.000 0.000 0.012 0.001 {hasattr.*} - 8 0.000 0.000 0.000 0.000 {range.*} - 4 0.000 0.000 0.000 0.000 {sys.exc_info.*} + 12 0.000 0.000 0.012 0.001 {hasattr} + 8 0.000 0.000 0.000 0.000 {range} + 4 0.000 0.000 0.000 0.000 {sys.exc_info} """ From afa at codespeak.net Tue Dec 7 18:11:05 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 18:11:05 +0100 (CET) Subject: [pypy-svn] r79872 - in pypy/branch/fast-forward/pypy/module/_weakref: . test Message-ID: <20101207171105.63560282BD4@codespeak.net> Author: afa Date: Tue Dec 7 18:11:03 2010 New Revision: 79872 Modified: pypy/branch/fast-forward/pypy/module/_weakref/interp__weakref.py pypy/branch/fast-forward/pypy/module/_weakref/test/test_weakref.py Log: weakref.ref accepts additional keyword arguments, but no extra positional arguments. Modified: pypy/branch/fast-forward/pypy/module/_weakref/interp__weakref.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_weakref/interp__weakref.py (original) +++ pypy/branch/fast-forward/pypy/module/_weakref/interp__weakref.py Tue Dec 7 18:11:03 2010 @@ -144,12 +144,21 @@ def descr__new__weakref(space, w_subtype, w_obj, w_callable=None, __args__=None): + if __args__.arguments_w: + raise OperationError(space.w_TypeError, space.wrap( + "__new__ expected at most 2 arguments")) lifeline = w_obj.getweakref() if lifeline is None: lifeline = WeakrefLifeline(space) w_obj.setweakref(space, lifeline) return lifeline.get_or_make_weakref(space, w_subtype, w_obj, w_callable) +def descr__init__weakref(space, self, w_obj, w_callable=None, + __args__=None): + if __args__.arguments_w: + raise OperationError(space.w_TypeError, space.wrap( + "__init__ expected at most 2 arguments")) + def descr__eq__(space, ref1, w_ref2): if not isinstance(w_ref2, W_Weakref): return space.w_NotImplemented @@ -186,6 +195,9 @@ __new__ = interp2app(descr__new__weakref, unwrap_spec=[ObjSpace, W_Root, W_Root, W_Root, Arguments]), + __init__ = interp2app(descr__init__weakref, + unwrap_spec=[ObjSpace, W_Weakref, W_Root, W_Root, + Arguments]), __eq__ = interp2app(descr__eq__, unwrap_spec=[ObjSpace, W_Weakref, W_Root]), __ne__ = interp2app(descr__ne__, Modified: pypy/branch/fast-forward/pypy/module/_weakref/test/test_weakref.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_weakref/test/test_weakref.py (original) +++ pypy/branch/fast-forward/pypy/module/_weakref/test/test_weakref.py Tue Dec 7 18:11:03 2010 @@ -423,3 +423,13 @@ del a1 gc.collect() assert ref1() is None + + def test_init(self): + import _weakref, gc + # Issue 3634 + # .__init__() doesn't check errors correctly + r = _weakref.ref(Exception) + raises(TypeError, r.__init__, 0, 0, 0, 0, 0) + # No exception should be raised here + gc.collect() + From afa at codespeak.net Tue Dec 7 18:23:11 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 18:23:11 +0100 (CET) Subject: [pypy-svn] r79873 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101207172311.93441282BD4@codespeak.net> Author: afa Date: Tue Dec 7 18:23:10 2010 New Revision: 79873 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_weakref.py Log: The exact order of weakrefs is certainly an implementation detail. Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_weakref.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_weakref.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_weakref.py Tue Dec 7 18:23:10 2010 @@ -738,9 +738,11 @@ self.assertEqual(weakref.getweakrefcount(o), 3) refs = weakref.getweakrefs(o) self.assertEqual(len(refs), 3) - self.assertTrue(r2 is refs[0]) - self.assertIn(r1, refs[1:]) - self.assertIn(r3, refs[1:]) + assert set(refs) == set((r1, r2, r3)) + if test_support.check_impl_detail(): + self.assertTrue(r2 is refs[0]) + self.assertIn(r1, refs[1:]) + self.assertIn(r3, refs[1:]) def test_subclass_refs_dont_conflate_callbacks(self): class MyRef(weakref.ref): From arigo at codespeak.net Tue Dec 7 18:28:59 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Dec 2010 18:28:59 +0100 (CET) Subject: [pypy-svn] r79874 - in pypy/trunk/pypy: interpreter interpreter/test module/_lsprof module/_lsprof/test Message-ID: <20101207172859.435E3282BD4@codespeak.net> Author: arigo Date: Tue Dec 7 18:28:57 2010 New Revision: 79874 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/interpreter/executioncontext.py pypy/trunk/pypy/interpreter/test/test_executioncontext.py pypy/trunk/pypy/module/_lsprof/interp_lsprof.py pypy/trunk/pypy/module/_lsprof/test/test_cprofile.py Log: Fix for issue423, with extra tests. Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Tue Dec 7 18:28:57 2010 @@ -862,14 +862,14 @@ def call_args_and_c_profile(self, frame, w_func, args): ec = self.getexecutioncontext() - ec.c_call_trace(frame, w_func) + ec.c_call_trace(frame, w_func, args) try: w_res = self.call_args(w_func, args) except OperationError, e: w_value = e.get_w_value(self) ec.c_exception_trace(frame, w_value) raise - ec.c_return_trace(frame, w_func) + ec.c_return_trace(frame, w_func, args) return w_res def call_method(self, w_obj, methname, *arg_w): Modified: pypy/trunk/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/executioncontext.py Tue Dec 7 18:28:57 2010 @@ -123,19 +123,30 @@ return lst # coroutine: I think this is all, folks! - def c_call_trace(self, frame, w_func): + def c_call_trace(self, frame, w_func, args=None): "Profile the call of a builtin function" - if self.profilefunc is None: - frame.is_being_profiled = False - else: - self._trace(frame, 'c_call', w_func) + self._c_call_return_trace(frame, w_func, args, 'c_call') - def c_return_trace(self, frame, w_retval): + def c_return_trace(self, frame, w_func, args=None): "Profile the return from a builtin function" + self._c_call_return_trace(frame, w_func, args, 'c_return') + + def _c_call_return_trace(self, frame, w_func, args, event): if self.profilefunc is None: frame.is_being_profiled = False else: - self._trace(frame, 'c_return', w_retval) + # undo the effect of the CALL_METHOD bytecode, which would be + # that even on a built-in method call like '[].append()', + # w_func is actually the unbound function 'append'. + from pypy.interpreter.function import FunctionWithFixedCode + if isinstance(w_func, FunctionWithFixedCode) and args is not None: + w_firstarg = args.firstarg() + if w_firstarg is not None: + from pypy.interpreter.function import descr_function_get + w_func = descr_function_get(self.space, w_func, w_firstarg, + self.space.type(w_firstarg)) + # + self._trace(frame, event, w_func) def c_exception_trace(self, frame, w_exc): "Profile function called upon OperationError." Modified: pypy/trunk/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/trunk/pypy/interpreter/test/test_executioncontext.py Tue Dec 7 18:28:57 2010 @@ -7,6 +7,10 @@ class TestExecutionContext: + keywords = {} + + def setup_class(cls): + cls.space = gettestobjspace(**cls.keywords) def test_action(self): @@ -77,29 +81,43 @@ assert l == ['call', 'return', 'call', 'return'] def test_llprofile_c_call(self): + from pypy.interpreter.function import Function, Method l = [] + seen = [] + space = self.space - def profile_func(space, w_arg, frame, event, w_aarg): + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) + if event == 'c_call': + seen.append(w_func) - space = self.space - space.getexecutioncontext().setllprofile(profile_func, space.w_None) - - def check_snippet(snippet): + def check_snippet(snippet, expected_c_call): + del l[:] + del seen[:] + space.getexecutioncontext().setllprofile(profile_func, + space.w_None) space.appexec([], """(): %s return """ % snippet) space.getexecutioncontext().setllprofile(None, None) assert l == ['call', 'return', 'call', 'c_call', 'c_return', 'return'] - - check_snippet('l = []; l.append(42)') - check_snippet('max(1, 2)') - check_snippet('args = (1, 2); max(*args)') - check_snippet('max(1, 2, **{})') - check_snippet('args = (1, 2); max(*args, **{})') - check_snippet('abs(val=0)') + if isinstance(seen[0], Method): + found = 'method %s of %s' % ( + seen[0].w_function.name, + seen[0].w_class.getname(space, '?')) + else: + assert isinstance(seen[0], Function) + found = 'builtin %s' % seen[0].name + assert found == expected_c_call + + check_snippet('l = []; l.append(42)', 'method append of list') + check_snippet('max(1, 2)', 'builtin max') + check_snippet('args = (1, 2); max(*args)', 'builtin max') + check_snippet('max(1, 2, **{})', 'builtin max') + check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') + check_snippet('abs(val=0)', 'builtin abs') def test_llprofile_c_exception(self): l = [] @@ -243,6 +261,13 @@ """) +class TestExecutionContextWithCallLikelyBuiltin(TestExecutionContext): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True} + +class TestExecutionContextWithCallMethod(TestExecutionContext): + keywords = {'objspace.opcodes.CALL_METHOD': True} + + class AppTestDelNotBlocked: def setup_method(self, meth): Modified: pypy/trunk/pypy/module/_lsprof/interp_lsprof.py ============================================================================== --- pypy/trunk/pypy/module/_lsprof/interp_lsprof.py (original) +++ pypy/trunk/pypy/module/_lsprof/interp_lsprof.py Tue Dec 7 18:28:57 2010 @@ -163,8 +163,11 @@ if isinstance(w_arg, Method): w_function = w_arg.w_function class_name = w_arg.w_class.getname(space, '?') - assert isinstance(w_function, Function) - return "{method '%s' of '%s' objects}" % (w_function.name, class_name) + if isinstance(w_function, Function): + name = w_function.name + else: + name = '?' + return "{method '%s' of '%s' objects}" % (name, class_name) elif isinstance(w_arg, Function): if w_arg.w_module is None: module = '' @@ -176,7 +179,8 @@ module += '.' return '{%s%s}' % (module, w_arg.name) else: - return '{!!!unknown!!!}' + class_name = w_arg.w_class.getname(space, '?') + return "{'%s' object}" % (class_name,) def lsprof_call(space, w_self, frame, event, w_arg): assert isinstance(w_self, W_Profiler) Modified: pypy/trunk/pypy/module/_lsprof/test/test_cprofile.py ============================================================================== --- pypy/trunk/pypy/module/_lsprof/test/test_cprofile.py (original) +++ pypy/trunk/pypy/module/_lsprof/test/test_cprofile.py Tue Dec 7 18:28:57 2010 @@ -150,7 +150,7 @@ sys.path.pop(0) -class AppTestDifferentBytecode(AppTestCProfile): +class AppTestWithDifferentBytecodes(AppTestCProfile): keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True, 'objspace.opcodes.CALL_METHOD': True} From arigo at codespeak.net Tue Dec 7 18:45:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Tue, 7 Dec 2010 18:45:17 +0100 (CET) Subject: [pypy-svn] r79875 - pypy/trunk/pypy/module/_lsprof Message-ID: <20101207174517.2C4CC282BAD@codespeak.net> Author: arigo Date: Tue Dec 7 18:45:15 2010 New Revision: 79875 Modified: pypy/trunk/pypy/module/_lsprof/interp_lsprof.py Log: Translation fix. Modified: pypy/trunk/pypy/module/_lsprof/interp_lsprof.py ============================================================================== --- pypy/trunk/pypy/module/_lsprof/interp_lsprof.py (original) +++ pypy/trunk/pypy/module/_lsprof/interp_lsprof.py Tue Dec 7 18:45:15 2010 @@ -179,7 +179,7 @@ module += '.' return '{%s%s}' % (module, w_arg.name) else: - class_name = w_arg.w_class.getname(space, '?') + class_name = space.type(w_arg).getname(space, '?') return "{'%s' object}" % (class_name,) def lsprof_call(space, w_self, frame, event, w_arg): From david at codespeak.net Tue Dec 7 20:39:34 2010 From: david at codespeak.net (david at codespeak.net) Date: Tue, 7 Dec 2010 20:39:34 +0100 (CET) Subject: [pypy-svn] r79877 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101207193934.E262E282BDC@codespeak.net> Author: david Date: Tue Dec 7 20:39:33 2010 New Revision: 79877 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: Improve guard_nonnull_class and some smaller fixes Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Tue Dec 7 20:39:33 2010 @@ -200,7 +200,7 @@ def _emit_guard(self, op, regalloc, fcond, save_exc=False): descr = op.getdescr() assert isinstance(descr, AbstractFailDescr) - if hasattr(op, 'getfailargs'): + if not we_are_translated() and hasattr(op, 'getfailargs'): print 'Failargs: ', op.getfailargs() self.mc.ensure_can_fit(self.guard_size) @@ -266,8 +266,13 @@ def emit_op_guard_nonnull_class(self, op, regalloc, fcond): locs = self._prepare_guard_class(op, regalloc, fcond) + offset = self.cpu.vtable_offset + self.mc.CMP_ri(locs[0].value, 0) - self._emit_guard(op, regalloc, c.NE) + if offset is not None: + self.mc.ADD_ri(r.pc.value, r.pc.value, 2*WORD, cond=c.EQ) + else: + raise NotImplementedError self._cmp_guard_class(op, locs, regalloc, fcond) return fcond @@ -326,6 +331,8 @@ adr = rffi.cast(lltype.Signed, op.getarg(0).getint()) args = op.getarglist()[1:] cond = self._emit_call(adr, args, regalloc, fcond, save_all_regs, op.result) + if op.result: + regalloc.possibly_free_var(op.result) descr = op.getdescr() #XXX Hack, Hack, Hack @@ -607,12 +614,13 @@ basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) - imm_ofs = self._check_imm_arg(ofs_length) + ofs_box = ConstInt(ofs_length) + imm_ofs = self._check_imm_arg(ofs_box) if imm_ofs: - l1 = regalloc.make_sure_var_in_reg(ConstInt(ofs_length), boxes) + l1 = regalloc.make_sure_var_in_reg(ofs_box, boxes) else: - l1, box1 = self._ensure_value_is_boxed(ConstInt(ofs_length), regalloc, boxes) + l1, box1 = self._ensure_value_is_boxed(ofs_box, regalloc, boxes) boxes.append(box1) regalloc.possibly_free_vars(boxes) @@ -788,12 +796,13 @@ boxes.append(op.result) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) - imm_ofs = self._check_imm_arg(ofs_length) + ofs_box = ConstInt(ofs_length) + imm_ofs = self._check_imm_arg(ofs_box) if imm_ofs: - l1 = regalloc.make_sure_var_in_reg(ConstInt(ofs_length), boxes) + l1 = regalloc.make_sure_var_in_reg(ofs_box, boxes) else: - l1, box1 = self._ensure_value_is_boxed(ConstInt(ofs_length), regalloc, boxes) + l1, box1 = self._ensure_value_is_boxed(ofs_box, regalloc, boxes) boxes.append(box1) regalloc.possibly_free_vars(boxes) @@ -900,6 +909,7 @@ # check value t = TempBox() + #XXX is this correct? resloc = regalloc.force_allocate_reg(resbox) loc = regalloc.force_allocate_reg(t) self.mc.gen_load_int(loc.value, value) @@ -1024,12 +1034,11 @@ self._emit_call(self.malloc_func_addr, [tempbox], regalloc, result=res_v) loc = regalloc.make_sure_var_in_reg(v, [res_v]) - regalloc.possibly_free_var(v) - regalloc.possibly_free_var(tempbox) - - # XXX combine with emit_op_setfield_gc operation base_loc = regalloc.loc(res_v) value_loc = regalloc.loc(v) + regalloc.possibly_free_vars([v, res_v, tempbox]) + + # XXX combine with emit_op_setfield_gc operation size = scale * 2 ofs = ofs_length if size == 4: From agaynor at codespeak.net Tue Dec 7 22:47:44 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 7 Dec 2010 22:47:44 +0100 (CET) Subject: [pypy-svn] r79878 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20101207214744.1BE94282BAD@codespeak.net> Author: agaynor Date: Tue Dec 7 22:47:41 2010 New Revision: 79878 Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py Log: Added support for using None in str.{index,find,rfind}. Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/stringobject.py Tue Dec 7 22:47:41 2010 @@ -44,7 +44,7 @@ from pypy.objspace.std.unicodetype import plain_str2unicode return plain_str2unicode(space, w_self._value) -def _is_generic(space, w_self, fun): +def _is_generic(space, w_self, fun): v = w_self._value if len(v) == 0: return space.w_False @@ -64,7 +64,7 @@ return chr(o) else: return ch - + def _lower(ch): if ch.isupper(): o = ord(ch) + 32 @@ -168,7 +168,7 @@ return space.wrap("".join(res)) - + def str_capitalize__String(space, w_self): input = w_self._value buffer = [' '] * len(input) @@ -189,7 +189,7 @@ buffer[i] = ch return space.wrap("".join(buffer)) - + def str_title__String(space, w_self): input = w_self._value buffer = [' '] * len(input) @@ -255,7 +255,7 @@ res_w.append(sliced(space, value, start, next, w_self)) start = next + bylen maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - + res_w.append(sliced(space, value, start, len(value), w_self)) return space.newlist(res_w) @@ -318,7 +318,7 @@ res_w.append(sliced(space, value, 0, end, w_self)) res_w.reverse() return space.newlist(res_w) - + return func_with_new_name(fn, funcname) str_rsplit__String_String_ANY = make_rsplit_with_delim('str_rsplit__String_String_ANY', @@ -360,12 +360,12 @@ if len(fillchar) != 1: raise OperationError(space.w_TypeError, space.wrap("rjust() argument 2 must be a single character")) - + d = u_arg - len(u_self) if d>0: fillchar = fillchar[0] # annotator hint: it's a single character u_self = d * fillchar + u_self - + return space.wrap(u_self) @@ -381,12 +381,14 @@ if d>0: fillchar = fillchar[0] # annotator hint: it's a single character u_self += d * fillchar - + return space.wrap(u_self) def _convert_idx_params(space, w_self, w_sub, w_start, w_end, upper_bound=False): self = w_self._value sub = w_sub._value + if space.is_w(w_end, space.w_None): + w_end = space.len(w_self) if upper_bound: start = slicetype.adapt_bound(space, len(self), w_start) end = slicetype.adapt_bound(space, len(self), w_end) @@ -488,7 +490,7 @@ substrings_w.append(input[start:next]) start = next + sublen maxsplit -= 1 # NB. if it's already < 0, it stays < 0 - + substrings_w.append(input[start:]) try: @@ -498,11 +500,11 @@ ovfcheck(one + len(input)) except OverflowError: raise OperationError( - space.w_OverflowError, + space.w_OverflowError, space.wrap("replace string is too long")) - + return space.wrap(by.join(substrings_w)) - + def str_replace__String_ANY_ANY_ANY(space, w_self, w_sub, w_by, w_maxsplit): return _string_replace(space, w_self._value, space.buffer_w(w_sub).as_str(), @@ -520,38 +522,38 @@ "internal function called by str_xstrip methods" u_self = w_self._value u_chars = w_chars._value - + lpos = 0 rpos = len(u_self) - + if left: #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) while lpos < rpos and u_self[lpos] in u_chars: lpos += 1 - + if right: while rpos > lpos and u_self[rpos - 1] in u_chars: rpos -= 1 - + assert rpos >= lpos # annotator hint, don't remove return sliced(space, u_self, lpos, rpos, w_self) def _strip_none(space, w_self, left, right): "internal function called by str_xstrip methods" u_self = w_self._value - + lpos = 0 rpos = len(u_self) - + if left: #print "while %d < %d and -%s- in -%s-:"%(lpos, rpos, u_self[lpos],w_chars) while lpos < rpos and u_self[lpos].isspace(): lpos += 1 - + if right: while rpos > lpos and u_self[rpos - 1].isspace(): rpos -= 1 - + assert rpos >= lpos # annotator hint, don't remove return sliced(space, u_self, lpos, rpos, w_self) @@ -560,14 +562,14 @@ def str_strip__String_None(space, w_self, w_chars): return _strip_none(space, w_self, left=1, right=1) - + def str_rstrip__String_String(space, w_self, w_chars): return _strip(space, w_self, w_chars, left=0, right=1) def str_rstrip__String_None(space, w_self, w_chars): return _strip_none(space, w_self, left=0, right=1) - + def str_lstrip__String_String(space, w_self, w_chars): return _strip(space, w_self, w_chars, left=1, right=0) @@ -584,7 +586,7 @@ raise OperationError(space.w_TypeError, space.wrap("center() argument 2 must be a single character")) - d = u_arg - len(u_self) + d = u_arg - len(u_self) if d>0: offset = d//2 + (d & u_arg & 1) fillchar = fillchar[0] # annotator hint: it's a single character @@ -594,7 +596,7 @@ return wrapstr(space, u_centered) -def str_count__String_String_ANY_ANY(space, w_self, w_arg, w_start, w_end): +def str_count__String_String_ANY_ANY(space, w_self, w_arg, w_start, w_end): u_self, u_arg, u_start, u_end = _convert_idx_params(space, w_self, w_arg, w_start, w_end) return wrapint(space, u_self.count(u_arg, u_start, u_end)) @@ -614,7 +616,7 @@ w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, w_end) - suffix = space.str_w(w_suffix) + suffix = space.str_w(w_suffix) if stringendswith(u_self, suffix, start, end): return space.w_True return space.w_False @@ -637,12 +639,12 @@ if stringstartswith(u_self, prefix, start, end): return space.w_True return space.w_False - + def _tabindent(u_token, u_tabsize): "calculates distance behind the token to the next tabstop" - + distance = u_tabsize - if u_token: + if u_token: distance = 0 offset = len(u_token) @@ -654,33 +656,33 @@ offset -= 1 if offset == 0: break - + #the same like distance = len(u_token) - (offset + 1) #print '' % (offset, distance, u_tabsize, u_token) distance = (u_tabsize-distance) % u_tabsize if distance == 0: distance=u_tabsize - return distance - - -def str_expandtabs__String_ANY(space, w_self, w_tabsize): + return distance + + +def str_expandtabs__String_ANY(space, w_self, w_tabsize): u_self = w_self._value u_tabsize = space.int_w(w_tabsize) - + u_expanded = "" if u_self: split = u_self.split("\t") u_expanded =oldtoken = split.pop(0) - for token in split: + for token in split: #print "%d#%d -%s-" % (_tabindent(oldtoken,u_tabsize), u_tabsize, token) u_expanded += " " * _tabindent(oldtoken,u_tabsize) + token oldtoken = token - - return wrapstr(space, u_expanded) - - + + return wrapstr(space, u_expanded) + + def str_splitlines__String_ANY(space, w_self, w_keepends): u_keepends = space.int_w(w_keepends) # truth value, but type checked data = w_self._value @@ -728,7 +730,7 @@ for i in range(middle, width): buf[i] = input[start] start = start + 1 - + return space.wrap("".join(buf)) def str_w__String(space, w_str): @@ -745,7 +747,7 @@ if s1 < s2: return space.w_True else: - return space.w_False + return space.w_False def le__String_String(space, w_str1, w_str2): s1 = w_str1._value @@ -924,10 +926,10 @@ def str_translate__String_ANY_ANY(space, w_string, w_table, w_deletechars=''): """charfilter - unicode handling is not implemented - - Return a copy of the string where all characters occurring - in the optional argument deletechars are removed, and the - remaining characters have been mapped through the given translation table, + + Return a copy of the string where all characters occurring + in the optional argument deletechars are removed, and the + remaining characters have been mapped through the given translation table, which must be a string of length 256""" # XXX CPython accepts buffers, too, not sure what we should do Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py Tue Dec 7 22:47:41 2010 @@ -12,7 +12,7 @@ assert self.space.str_w(self.space.wrap("foo")) == "foo" def test_equality(self): - w = self.space.wrap + w = self.space.wrap assert self.space.eq_w(w('abc'), w('abc')) assert not self.space.eq_w(w('abc'), w('def')) @@ -175,7 +175,7 @@ assert "!Brown Fox".istitle() == True assert "Brow&&&&N Fox".istitle() == True assert "!Brow&&&&n Fox".istitle() == False - + def test_capitalize(self): assert "brown fox".capitalize() == "Brown fox" assert ' hello '.capitalize() == ' hello ' @@ -257,7 +257,7 @@ assert ''.zfill(3) == '000' assert '34'.zfill(1) == '34' assert '34'.zfill(4) == '0034' - + def test_center(self): s="a b" assert s.center(0) == "a b" @@ -277,7 +277,7 @@ assert 'abc'.center(5, '*') == '*abc*' # Python 2.4 raises(TypeError, 'abc'.center, 4, 'cba') assert ' abc'.center(7) == ' abc ' - + def test_count(self): assert "".count("x") ==0 assert "".count("") ==1 @@ -290,7 +290,7 @@ assert 'aaa'.count('a', 0, -1) == 2 assert 'aaa'.count('a', 0, -10) == 0 assert 'ababa'.count('aba') == 1 - + def test_startswith(self): assert 'ab'.startswith('ab') is True assert 'ab'.startswith('a') is True @@ -320,7 +320,7 @@ assert not 'hello'.startswith(('he', 'hel'), 0, 1) assert 'hello'.startswith(('he', 'hel'), 0, 2) raises(TypeError, 'hello'.startswith, (42,)) - + def test_endswith(self): assert 'ab'.endswith('ab') is True assert 'ab'.endswith('b') is True @@ -361,13 +361,13 @@ s = 'xy\t' assert s.expandtabs() =='xy ' - + s = '\txy\t' assert s.expandtabs() ==' xy ' assert s.expandtabs(1) ==' xy ' assert s.expandtabs(2) ==' xy ' assert s.expandtabs(3) ==' xy ' - + assert 'xy'.expandtabs() =='xy' assert ''.expandtabs() =='' @@ -398,12 +398,13 @@ # Split on \r and \r\n too assert '12\r34\r\n56'.splitlines() == ['12', '34', '56'] assert '12\r34\r\n56'.splitlines(1) == ['12\r', '34\r\n', '56'] - + def test_find(self): assert 'abcdefghiabc'.find('abc') == 0 assert 'abcdefghiabc'.find('abc', 1) == 9 assert 'abcdefghiabc'.find('def', 4) == -1 assert 'abcdef'.find('', 13) == -1 + assert 'abcdefg'.find('def', 5, None) == -1 def test_index(self): from sys import maxint @@ -412,6 +413,7 @@ assert 'abcdefghiabc'.index('abc') == 0 assert 'abcdefghiabc'.index('abc', 1) == 9 assert 'abcdefghiabc'.index('def', -4*maxint, 4*maxint) == 3 + assert 'abcdefgh'.index('def', 2, None) == 3 raises(ValueError, 'abcdefghiabc'.index, 'hib') raises(ValueError, 'abcdefghiab'.index, 'abc', 1) raises(ValueError, 'abcdefghi'.index, 'ghi', 8) @@ -427,6 +429,7 @@ assert 'abcdefghiabc'.rfind('abcz') == -1 assert 'abc'.rfind('', 0) == 3 assert 'abc'.rfind('', 3) == 3 + assert 'abcdefgh'.rfind('def', 2, None) == 3 def test_rindex(self): from sys import maxint @@ -507,7 +510,7 @@ if i == 2: return unicode("fooled you!") return self.it.next() - + f = ('a\n', 'b\n', 'c\n') got = " - ".join(OhPhooey(f)) assert got == unicode("a\n - b\n - fooled you! - c\n") @@ -545,7 +548,7 @@ assert "\t\t\b\b\n".isspace() == False assert "\t\t".isspace() == True assert "\t\t\r\r\n".isspace() == True - + def test_islower(self): assert "".islower() == False assert " ".islower() == False @@ -565,8 +568,8 @@ assert "!BBB".isupper() == True assert "bbb".isupper() == False assert "BBBbbb".isupper() == False - - + + def test_swapcase(self): assert "aaa AAA 111".swapcase() == "AAA aaa 111" assert "".swapcase() == "" @@ -581,7 +584,7 @@ tbl = ''.join(L) return tbl - + table = maketrans('abc', 'xyz') assert 'xyzxyz' == 'xyzabcdef'.translate(table, 'def') @@ -589,7 +592,7 @@ assert 'Abc' == 'abc'.translate(table) assert 'xyz' == 'xyz'.translate(table) assert 'yz' == 'xyz'.translate(table, 'x') - + raises(ValueError, 'xyz'.translate, 'too short', 'strip') raises(ValueError, 'xyz'.translate, 'too short') raises(ValueError, 'xyz'.translate, 'too long'*33) @@ -601,7 +604,7 @@ for i in iter("42"): l.append(i) assert l == ['4','2'] - + def test_repr(self): assert repr("") =="''" assert repr("a") =="'a'" @@ -636,7 +639,7 @@ def test_encode(self): assert 'hello'.encode() == 'hello' assert type('hello'.encode()) is str - + def test_hash(self): # check that we have the same hash as CPython for at least 31 bits # (but don't go checking CPython's special case -1) From agaynor at codespeak.net Tue Dec 7 22:53:45 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 7 Dec 2010 22:53:45 +0100 (CET) Subject: [pypy-svn] r79879 - pypy/branch/fast-forward/lib-python Message-ID: <20101207215345.0898A282BAD@codespeak.net> Author: agaynor Date: Tue Dec 7 22:53:44 2010 New Revision: 79879 Modified: pypy/branch/fast-forward/lib-python/TODO Log: This is done. Modified: pypy/branch/fast-forward/lib-python/TODO ============================================================================== --- pypy/branch/fast-forward/lib-python/TODO (original) +++ pypy/branch/fast-forward/lib-python/TODO Tue Dec 7 22:53:44 2010 @@ -22,7 +22,7 @@ @test_support.cpython_only -- (list|str|unicode|bytearray).(index|find) should accept None as indices +- (list|unicode|bytearray).(index|find) should accept None as indices - Fix fcntl.fcntl(fd, fcntl.F_NOTIFY, fcntl.DN_MULTISHOT) on 32bit platform. From afa at codespeak.net Tue Dec 7 23:52:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 23:52:59 +0100 (CET) Subject: [pypy-svn] r79880 - in pypy/branch/fast-forward/pypy/translator/c: src test Message-ID: <20101207225259.C8B77282BAD@codespeak.net> Author: afa Date: Tue Dec 7 23:52:57 2010 New Revision: 79880 Added: pypy/branch/fast-forward/pypy/translator/c/src/dtoa.c (contents, props changed) pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py (contents, props changed) Log: Add dtoa.c (shorter float representation) and try to use it in test functions. Very incomplete so far Added: pypy/branch/fast-forward/pypy/translator/c/src/dtoa.c ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/translator/c/src/dtoa.c Tue Dec 7 23:52:57 2010 @@ -0,0 +1,2931 @@ +/**************************************************************** + * + * The author of this software is David M. Gay. + * + * Copyright (c) 1991, 2000, 2001 by Lucent Technologies. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose without fee is hereby granted, provided that this entire notice + * is included in all copies of any software which is or includes a copy + * or modification of this software and in all copies of the supporting + * documentation for such software. + * + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR LUCENT MAKES ANY + * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY + * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. + * + ***************************************************************/ + +/**************************************************************** + * This is dtoa.c by David M. Gay, downloaded from + * http://www.netlib.org/fp/dtoa.c on April 15, 2009 and modified for + * inclusion into the Python core by Mark E. T. Dickinson and Eric V. Smith. + * + * Please remember to check http://www.netlib.org/fp regularly (and especially + * before any Python release) for bugfixes and updates. + * + * The major modifications from Gay's original code are as follows: + * + * 0. The original code has been specialized to Python's needs by removing + * many of the #ifdef'd sections. In particular, code to support VAX and + * IBM floating-point formats, hex NaNs, hex floats, locale-aware + * treatment of the decimal point, and setting of the inexact flag have + * been removed. + * + * 1. We use PyMem_Malloc and PyMem_Free in place of malloc and free. + * + * 2. The public functions strtod, dtoa and freedtoa all now have + * a _Py_dg_ prefix. + * + * 3. Instead of assuming that PyMem_Malloc always succeeds, we thread + * PyMem_Malloc failures through the code. The functions + * + * Balloc, multadd, s2b, i2b, mult, pow5mult, lshift, diff, d2b + * + * of return type *Bigint all return NULL to indicate a malloc failure. + * Similarly, rv_alloc and nrv_alloc (return type char *) return NULL on + * failure. bigcomp now has return type int (it used to be void) and + * returns -1 on failure and 0 otherwise. _Py_dg_dtoa returns NULL + * on failure. _Py_dg_strtod indicates failure due to malloc failure + * by returning -1.0, setting errno=ENOMEM and *se to s00. + * + * 4. The static variable dtoa_result has been removed. Callers of + * _Py_dg_dtoa are expected to call _Py_dg_freedtoa to free + * the memory allocated by _Py_dg_dtoa. + * + * 5. The code has been reformatted to better fit with Python's + * C style guide (PEP 7). + * + * 6. A bug in the memory allocation has been fixed: to avoid FREEing memory + * that hasn't been MALLOC'ed, private_mem should only be used when k <= + * Kmax. + * + * 7. _Py_dg_strtod has been modified so that it doesn't accept strings with + * leading whitespace. + * + ***************************************************************/ + +/* Please send bug reports for the original dtoa.c code to David M. Gay (dmg + * at acm dot org, with " at " changed at "@" and " dot " changed to "."). + * Please report bugs for this modified version using the Python issue tracker + * (http://bugs.python.org). */ + +/* On a machine with IEEE extended-precision registers, it is + * necessary to specify double-precision (53-bit) rounding precision + * before invoking strtod or dtoa. If the machine uses (the equivalent + * of) Intel 80x87 arithmetic, the call + * _control87(PC_53, MCW_PC); + * does this with many compilers. Whether this or another call is + * appropriate depends on the compiler; for this to work, it may be + * necessary to #include "float.h" or another system-dependent header + * file. + */ + +/* strtod for IEEE-, VAX-, and IBM-arithmetic machines. + * + * This strtod returns a nearest machine number to the input decimal + * string (or sets errno to ERANGE). With IEEE arithmetic, ties are + * broken by the IEEE round-even rule. Otherwise ties are broken by + * biased rounding (add half and chop). + * + * Inspired loosely by William D. Clinger's paper "How to Read Floating + * Point Numbers Accurately" [Proc. ACM SIGPLAN '90, pp. 92-101]. + * + * Modifications: + * + * 1. We only require IEEE, IBM, or VAX double-precision + * arithmetic (not IEEE double-extended). + * 2. We get by with floating-point arithmetic in a case that + * Clinger missed -- when we're computing d * 10^n + * for a small integer d and the integer n is not too + * much larger than 22 (the maximum integer k for which + * we can represent 10^k exactly), we may be able to + * compute (d*10^k) * 10^(e-k) with just one roundoff. + * 3. Rather than a bit-at-a-time adjustment of the binary + * result in the hard case, we use floating-point + * arithmetic to determine the adjustment to within + * one bit; only in really hard cases do we need to + * compute a second residual. + * 4. Because of 3., we don't need a large table of powers of 10 + * for ten-to-e (just some small tables, e.g. of 10^k + * for 0 <= k <= 22). + */ + +/* Linking of Python's #defines to Gay's #defines starts here. */ + +/* Begin PYPY hacks */ +/* #include "Python.h" */ +#define IEEE_8087 +#define HAVE_UINT32_T +#define HAVE_INT32_T +#define PY_UINT32_T int +#define PY_INT32_T int +#include +#include +#include +#define PyMem_Malloc PyObject_Malloc +#define PyMem_Free PyObject_Free +/* End PYPY hacks */ + + +/* if PY_NO_SHORT_FLOAT_REPR is defined, then don't even try to compile + the following code */ +#ifndef PY_NO_SHORT_FLOAT_REPR + +#include + +#define MALLOC PyMem_Malloc +#define FREE PyMem_Free + +/* This code should also work for ARM mixed-endian format on little-endian + machines, where doubles have byte order 45670123 (in increasing address + order, 0 being the least significant byte). */ +#ifdef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 +# define IEEE_8087 +#endif +#if defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) || \ + defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754) +# define IEEE_MC68k +#endif +#if defined(IEEE_8087) + defined(IEEE_MC68k) != 1 +#error "Exactly one of IEEE_8087 or IEEE_MC68k should be defined." +#endif + +/* The code below assumes that the endianness of integers matches the + endianness of the two 32-bit words of a double. Check this. */ +#if defined(WORDS_BIGENDIAN) && (defined(DOUBLE_IS_LITTLE_ENDIAN_IEEE754) || \ + defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754)) +#error "doubles and ints have incompatible endianness" +#endif + +#if !defined(WORDS_BIGENDIAN) && defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) +#error "doubles and ints have incompatible endianness" +#endif + + +#if defined(HAVE_UINT32_T) && defined(HAVE_INT32_T) +typedef PY_UINT32_T ULong; +typedef PY_INT32_T Long; +#else +#error "Failed to find an exact-width 32-bit integer type" +#endif + +#if defined(HAVE_UINT64_T) +#define ULLong PY_UINT64_T +#else +#undef ULLong +#endif + +#undef DEBUG +#ifdef Py_DEBUG +#define DEBUG +#endif + +/* End Python #define linking */ + +#ifdef DEBUG +#define Bug(x) {fprintf(stderr, "%s\n", x); exit(1);} +#endif + +#ifndef PRIVATE_MEM +#define PRIVATE_MEM 2304 +#endif +#define PRIVATE_mem ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) +static double private_mem[PRIVATE_mem], *pmem_next = private_mem; + +#ifdef __cplusplus +extern "C" { +#endif + +typedef union { double d; ULong L[2]; } U; + +#ifdef IEEE_8087 +#define word0(x) (x)->L[1] +#define word1(x) (x)->L[0] +#else +#define word0(x) (x)->L[0] +#define word1(x) (x)->L[1] +#endif +#define dval(x) (x)->d + +#ifndef STRTOD_DIGLIM +#define STRTOD_DIGLIM 40 +#endif + +/* maximum permitted exponent value for strtod; exponents larger than + MAX_ABS_EXP in absolute value get truncated to +-MAX_ABS_EXP. MAX_ABS_EXP + should fit into an int. */ +#ifndef MAX_ABS_EXP +#define MAX_ABS_EXP 19999U +#endif + +/* The following definition of Storeinc is appropriate for MIPS processors. + * An alternative that might be better on some machines is + * #define Storeinc(a,b,c) (*a++ = b << 16 | c & 0xffff) + */ +#if defined(IEEE_8087) +#define Storeinc(a,b,c) (((unsigned short *)a)[1] = (unsigned short)b, \ + ((unsigned short *)a)[0] = (unsigned short)c, a++) +#else +#define Storeinc(a,b,c) (((unsigned short *)a)[0] = (unsigned short)b, \ + ((unsigned short *)a)[1] = (unsigned short)c, a++) +#endif + +/* #define P DBL_MANT_DIG */ +/* Ten_pmax = floor(P*log(2)/log(5)) */ +/* Bletch = (highest power of 2 < DBL_MAX_10_EXP) / 16 */ +/* Quick_max = floor((P-1)*log(FLT_RADIX)/log(10) - 1) */ +/* Int_max = floor(P*log(FLT_RADIX)/log(10) - 1) */ + +#define Exp_shift 20 +#define Exp_shift1 20 +#define Exp_msk1 0x100000 +#define Exp_msk11 0x100000 +#define Exp_mask 0x7ff00000 +#define P 53 +#define Nbits 53 +#define Bias 1023 +#define Emax 1023 +#define Emin (-1022) +#define Etiny (-1074) /* smallest denormal is 2**Etiny */ +#define Exp_1 0x3ff00000 +#define Exp_11 0x3ff00000 +#define Ebits 11 +#define Frac_mask 0xfffff +#define Frac_mask1 0xfffff +#define Ten_pmax 22 +#define Bletch 0x10 +#define Bndry_mask 0xfffff +#define Bndry_mask1 0xfffff +#define Sign_bit 0x80000000 +#define Log2P 1 +#define Tiny0 0 +#define Tiny1 1 +#define Quick_max 14 +#define Int_max 14 + +#ifndef Flt_Rounds +#ifdef FLT_ROUNDS +#define Flt_Rounds FLT_ROUNDS +#else +#define Flt_Rounds 1 +#endif +#endif /*Flt_Rounds*/ + +#define Rounding Flt_Rounds + +#define Big0 (Frac_mask1 | Exp_msk1*(DBL_MAX_EXP+Bias-1)) +#define Big1 0xffffffff + +/* struct BCinfo is used to pass information from _Py_dg_strtod to bigcomp */ + +typedef struct BCinfo BCinfo; +struct +BCinfo { + int e0, nd, nd0, scale; +}; + +#define FFFFFFFF 0xffffffffUL + +#define Kmax 7 + +/* struct Bigint is used to represent arbitrary-precision integers. These + integers are stored in sign-magnitude format, with the magnitude stored as + an array of base 2**32 digits. Bigints are always normalized: if x is a + Bigint then x->wds >= 1, and either x->wds == 1 or x[wds-1] is nonzero. + + The Bigint fields are as follows: + + - next is a header used by Balloc and Bfree to keep track of lists + of freed Bigints; it's also used for the linked list of + powers of 5 of the form 5**2**i used by pow5mult. + - k indicates which pool this Bigint was allocated from + - maxwds is the maximum number of words space was allocated for + (usually maxwds == 2**k) + - sign is 1 for negative Bigints, 0 for positive. The sign is unused + (ignored on inputs, set to 0 on outputs) in almost all operations + involving Bigints: a notable exception is the diff function, which + ignores signs on inputs but sets the sign of the output correctly. + - wds is the actual number of significant words + - x contains the vector of words (digits) for this Bigint, from least + significant (x[0]) to most significant (x[wds-1]). +*/ + +struct +Bigint { + struct Bigint *next; + int k, maxwds, sign, wds; + ULong x[1]; +}; + +typedef struct Bigint Bigint; + +#ifndef Py_USING_MEMORY_DEBUGGER + +/* Memory management: memory is allocated from, and returned to, Kmax+1 pools + of memory, where pool k (0 <= k <= Kmax) is for Bigints b with b->maxwds == + 1 << k. These pools are maintained as linked lists, with freelist[k] + pointing to the head of the list for pool k. + + On allocation, if there's no free slot in the appropriate pool, MALLOC is + called to get more memory. This memory is not returned to the system until + Python quits. There's also a private memory pool that's allocated from + in preference to using MALLOC. + + For Bigints with more than (1 << Kmax) digits (which implies at least 1233 + decimal digits), memory is directly allocated using MALLOC, and freed using + FREE. + + XXX: it would be easy to bypass this memory-management system and + translate each call to Balloc into a call to PyMem_Malloc, and each + Bfree to PyMem_Free. Investigate whether this has any significant + performance on impact. */ + +static Bigint *freelist[Kmax+1]; + +/* Allocate space for a Bigint with up to 1<next; + else { + x = 1 << k; + len = (sizeof(Bigint) + (x-1)*sizeof(ULong) + sizeof(double) - 1) + /sizeof(double); + if (k <= Kmax && pmem_next - private_mem + len <= PRIVATE_mem) { + rv = (Bigint*)pmem_next; + pmem_next += len; + } + else { + rv = (Bigint*)MALLOC(len*sizeof(double)); + if (rv == NULL) + return NULL; + } + rv->k = k; + rv->maxwds = x; + } + rv->sign = rv->wds = 0; + return rv; +} + +/* Free a Bigint allocated with Balloc */ + +static void +Bfree(Bigint *v) +{ + if (v) { + if (v->k > Kmax) + FREE((void*)v); + else { + v->next = freelist[v->k]; + freelist[v->k] = v; + } + } +} + +#else + +/* Alternative versions of Balloc and Bfree that use PyMem_Malloc and + PyMem_Free directly in place of the custom memory allocation scheme above. + These are provided for the benefit of memory debugging tools like + Valgrind. */ + +/* Allocate space for a Bigint with up to 1<k = k; + rv->maxwds = x; + rv->sign = rv->wds = 0; + return rv; +} + +/* Free a Bigint allocated with Balloc */ + +static void +Bfree(Bigint *v) +{ + if (v) { + FREE((void*)v); + } +} + +#endif /* Py_USING_MEMORY_DEBUGGER */ + +#define Bcopy(x,y) memcpy((char *)&x->sign, (char *)&y->sign, \ + y->wds*sizeof(Long) + 2*sizeof(int)) + +/* Multiply a Bigint b by m and add a. Either modifies b in place and returns + a pointer to the modified b, or Bfrees b and returns a pointer to a copy. + On failure, return NULL. In this case, b will have been already freed. */ + +static Bigint * +multadd(Bigint *b, int m, int a) /* multiply by m and add a */ +{ + int i, wds; +#ifdef ULLong + ULong *x; + ULLong carry, y; +#else + ULong carry, *x, y; + ULong xi, z; +#endif + Bigint *b1; + + wds = b->wds; + x = b->x; + i = 0; + carry = a; + do { +#ifdef ULLong + y = *x * (ULLong)m + carry; + carry = y >> 32; + *x++ = (ULong)(y & FFFFFFFF); +#else + xi = *x; + y = (xi & 0xffff) * m + carry; + z = (xi >> 16) * m + (y >> 16); + carry = z >> 16; + *x++ = (z << 16) + (y & 0xffff); +#endif + } + while(++i < wds); + if (carry) { + if (wds >= b->maxwds) { + b1 = Balloc(b->k+1); + if (b1 == NULL){ + Bfree(b); + return NULL; + } + Bcopy(b1, b); + Bfree(b); + b = b1; + } + b->x[wds++] = (ULong)carry; + b->wds = wds; + } + return b; +} + +/* convert a string s containing nd decimal digits (possibly containing a + decimal separator at position nd0, which is ignored) to a Bigint. This + function carries on where the parsing code in _Py_dg_strtod leaves off: on + entry, y9 contains the result of converting the first 9 digits. Returns + NULL on failure. */ + +static Bigint * +s2b(const char *s, int nd0, int nd, ULong y9) +{ + Bigint *b; + int i, k; + Long x, y; + + x = (nd + 8) / 9; + for(k = 0, y = 1; x > y; y <<= 1, k++) ; + b = Balloc(k); + if (b == NULL) + return NULL; + b->x[0] = y9; + b->wds = 1; + + if (nd <= 9) + return b; + + s += 9; + for (i = 9; i < nd0; i++) { + b = multadd(b, 10, *s++ - '0'); + if (b == NULL) + return NULL; + } + s++; + for(; i < nd; i++) { + b = multadd(b, 10, *s++ - '0'); + if (b == NULL) + return NULL; + } + return b; +} + +/* count leading 0 bits in the 32-bit integer x. */ + +static int +hi0bits(ULong x) +{ + int k = 0; + + if (!(x & 0xffff0000)) { + k = 16; + x <<= 16; + } + if (!(x & 0xff000000)) { + k += 8; + x <<= 8; + } + if (!(x & 0xf0000000)) { + k += 4; + x <<= 4; + } + if (!(x & 0xc0000000)) { + k += 2; + x <<= 2; + } + if (!(x & 0x80000000)) { + k++; + if (!(x & 0x40000000)) + return 32; + } + return k; +} + +/* count trailing 0 bits in the 32-bit integer y, and shift y right by that + number of bits. */ + +static int +lo0bits(ULong *y) +{ + int k; + ULong x = *y; + + if (x & 7) { + if (x & 1) + return 0; + if (x & 2) { + *y = x >> 1; + return 1; + } + *y = x >> 2; + return 2; + } + k = 0; + if (!(x & 0xffff)) { + k = 16; + x >>= 16; + } + if (!(x & 0xff)) { + k += 8; + x >>= 8; + } + if (!(x & 0xf)) { + k += 4; + x >>= 4; + } + if (!(x & 0x3)) { + k += 2; + x >>= 2; + } + if (!(x & 1)) { + k++; + x >>= 1; + if (!x) + return 32; + } + *y = x; + return k; +} + +/* convert a small nonnegative integer to a Bigint */ + +static Bigint * +i2b(int i) +{ + Bigint *b; + + b = Balloc(1); + if (b == NULL) + return NULL; + b->x[0] = i; + b->wds = 1; + return b; +} + +/* multiply two Bigints. Returns a new Bigint, or NULL on failure. Ignores + the signs of a and b. */ + +static Bigint * +mult(Bigint *a, Bigint *b) +{ + Bigint *c; + int k, wa, wb, wc; + ULong *x, *xa, *xae, *xb, *xbe, *xc, *xc0; + ULong y; +#ifdef ULLong + ULLong carry, z; +#else + ULong carry, z; + ULong z2; +#endif + + if ((!a->x[0] && a->wds == 1) || (!b->x[0] && b->wds == 1)) { + c = Balloc(0); + if (c == NULL) + return NULL; + c->wds = 1; + c->x[0] = 0; + return c; + } + + if (a->wds < b->wds) { + c = a; + a = b; + b = c; + } + k = a->k; + wa = a->wds; + wb = b->wds; + wc = wa + wb; + if (wc > a->maxwds) + k++; + c = Balloc(k); + if (c == NULL) + return NULL; + for(x = c->x, xa = x + wc; x < xa; x++) + *x = 0; + xa = a->x; + xae = xa + wa; + xb = b->x; + xbe = xb + wb; + xc0 = c->x; +#ifdef ULLong + for(; xb < xbe; xc0++) { + if ((y = *xb++)) { + x = xa; + xc = xc0; + carry = 0; + do { + z = *x++ * (ULLong)y + *xc + carry; + carry = z >> 32; + *xc++ = (ULong)(z & FFFFFFFF); + } + while(x < xae); + *xc = (ULong)carry; + } + } +#else + for(; xb < xbe; xb++, xc0++) { + if (y = *xb & 0xffff) { + x = xa; + xc = xc0; + carry = 0; + do { + z = (*x & 0xffff) * y + (*xc & 0xffff) + carry; + carry = z >> 16; + z2 = (*x++ >> 16) * y + (*xc >> 16) + carry; + carry = z2 >> 16; + Storeinc(xc, z2, z); + } + while(x < xae); + *xc = carry; + } + if (y = *xb >> 16) { + x = xa; + xc = xc0; + carry = 0; + z2 = *xc; + do { + z = (*x & 0xffff) * y + (*xc >> 16) + carry; + carry = z >> 16; + Storeinc(xc, z, z2); + z2 = (*x++ >> 16) * y + (*xc & 0xffff) + carry; + carry = z2 >> 16; + } + while(x < xae); + *xc = z2; + } + } +#endif + for(xc0 = c->x, xc = xc0 + wc; wc > 0 && !*--xc; --wc) ; + c->wds = wc; + return c; +} + +#ifndef Py_USING_MEMORY_DEBUGGER + +/* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + +static Bigint *p5s; + +/* multiply the Bigint b by 5**k. Returns a pointer to the result, or NULL on + failure; if the returned pointer is distinct from b then the original + Bigint b will have been Bfree'd. Ignores the sign of b. */ + +static Bigint * +pow5mult(Bigint *b, int k) +{ + Bigint *b1, *p5, *p51; + int i; + static int p05[3] = { 5, 25, 125 }; + + if ((i = k & 3)) { + b = multadd(b, p05[i-1], 0); + if (b == NULL) + return NULL; + } + + if (!(k >>= 2)) + return b; + p5 = p5s; + if (!p5) { + /* first time */ + p5 = i2b(625); + if (p5 == NULL) { + Bfree(b); + return NULL; + } + p5s = p5; + p5->next = 0; + } + for(;;) { + if (k & 1) { + b1 = mult(b, p5); + Bfree(b); + b = b1; + if (b == NULL) + return NULL; + } + if (!(k >>= 1)) + break; + p51 = p5->next; + if (!p51) { + p51 = mult(p5,p5); + if (p51 == NULL) { + Bfree(b); + return NULL; + } + p51->next = 0; + p5->next = p51; + } + p5 = p51; + } + return b; +} + +#else + +/* Version of pow5mult that doesn't cache powers of 5. Provided for + the benefit of memory debugging tools like Valgrind. */ + +static Bigint * +pow5mult(Bigint *b, int k) +{ + Bigint *b1, *p5, *p51; + int i; + static int p05[3] = { 5, 25, 125 }; + + if ((i = k & 3)) { + b = multadd(b, p05[i-1], 0); + if (b == NULL) + return NULL; + } + + if (!(k >>= 2)) + return b; + p5 = i2b(625); + if (p5 == NULL) { + Bfree(b); + return NULL; + } + + for(;;) { + if (k & 1) { + b1 = mult(b, p5); + Bfree(b); + b = b1; + if (b == NULL) { + Bfree(p5); + return NULL; + } + } + if (!(k >>= 1)) + break; + p51 = mult(p5, p5); + Bfree(p5); + p5 = p51; + if (p5 == NULL) { + Bfree(b); + return NULL; + } + } + Bfree(p5); + return b; +} + +#endif /* Py_USING_MEMORY_DEBUGGER */ + +/* shift a Bigint b left by k bits. Return a pointer to the shifted result, + or NULL on failure. If the returned pointer is distinct from b then the + original b will have been Bfree'd. Ignores the sign of b. */ + +static Bigint * +lshift(Bigint *b, int k) +{ + int i, k1, n, n1; + Bigint *b1; + ULong *x, *x1, *xe, z; + + if (!k || (!b->x[0] && b->wds == 1)) + return b; + + n = k >> 5; + k1 = b->k; + n1 = n + b->wds + 1; + for(i = b->maxwds; n1 > i; i <<= 1) + k1++; + b1 = Balloc(k1); + if (b1 == NULL) { + Bfree(b); + return NULL; + } + x1 = b1->x; + for(i = 0; i < n; i++) + *x1++ = 0; + x = b->x; + xe = x + b->wds; + if (k &= 0x1f) { + k1 = 32 - k; + z = 0; + do { + *x1++ = *x << k | z; + z = *x++ >> k1; + } + while(x < xe); + if ((*x1 = z)) + ++n1; + } + else do + *x1++ = *x++; + while(x < xe); + b1->wds = n1 - 1; + Bfree(b); + return b1; +} + +/* Do a three-way compare of a and b, returning -1 if a < b, 0 if a == b and + 1 if a > b. Ignores signs of a and b. */ + +static int +cmp(Bigint *a, Bigint *b) +{ + ULong *xa, *xa0, *xb, *xb0; + int i, j; + + i = a->wds; + j = b->wds; +#ifdef DEBUG + if (i > 1 && !a->x[i-1]) + Bug("cmp called with a->x[a->wds-1] == 0"); + if (j > 1 && !b->x[j-1]) + Bug("cmp called with b->x[b->wds-1] == 0"); +#endif + if (i -= j) + return i; + xa0 = a->x; + xa = xa0 + j; + xb0 = b->x; + xb = xb0 + j; + for(;;) { + if (*--xa != *--xb) + return *xa < *xb ? -1 : 1; + if (xa <= xa0) + break; + } + return 0; +} + +/* Take the difference of Bigints a and b, returning a new Bigint. Returns + NULL on failure. The signs of a and b are ignored, but the sign of the + result is set appropriately. */ + +static Bigint * +diff(Bigint *a, Bigint *b) +{ + Bigint *c; + int i, wa, wb; + ULong *xa, *xae, *xb, *xbe, *xc; +#ifdef ULLong + ULLong borrow, y; +#else + ULong borrow, y; + ULong z; +#endif + + i = cmp(a,b); + if (!i) { + c = Balloc(0); + if (c == NULL) + return NULL; + c->wds = 1; + c->x[0] = 0; + return c; + } + if (i < 0) { + c = a; + a = b; + b = c; + i = 1; + } + else + i = 0; + c = Balloc(a->k); + if (c == NULL) + return NULL; + c->sign = i; + wa = a->wds; + xa = a->x; + xae = xa + wa; + wb = b->wds; + xb = b->x; + xbe = xb + wb; + xc = c->x; + borrow = 0; +#ifdef ULLong + do { + y = (ULLong)*xa++ - *xb++ - borrow; + borrow = y >> 32 & (ULong)1; + *xc++ = (ULong)(y & FFFFFFFF); + } + while(xb < xbe); + while(xa < xae) { + y = *xa++ - borrow; + borrow = y >> 32 & (ULong)1; + *xc++ = (ULong)(y & FFFFFFFF); + } +#else + do { + y = (*xa & 0xffff) - (*xb & 0xffff) - borrow; + borrow = (y & 0x10000) >> 16; + z = (*xa++ >> 16) - (*xb++ >> 16) - borrow; + borrow = (z & 0x10000) >> 16; + Storeinc(xc, z, y); + } + while(xb < xbe); + while(xa < xae) { + y = (*xa & 0xffff) - borrow; + borrow = (y & 0x10000) >> 16; + z = (*xa++ >> 16) - borrow; + borrow = (z & 0x10000) >> 16; + Storeinc(xc, z, y); + } +#endif + while(!*--xc) + wa--; + c->wds = wa; + return c; +} + +/* Given a positive normal double x, return the difference between x and the + next double up. Doesn't give correct results for subnormals. */ + +static double +ulp(U *x) +{ + Long L; + U u; + + L = (word0(x) & Exp_mask) - (P-1)*Exp_msk1; + word0(&u) = L; + word1(&u) = 0; + return dval(&u); +} + +/* Convert a Bigint to a double plus an exponent */ + +static double +b2d(Bigint *a, int *e) +{ + ULong *xa, *xa0, w, y, z; + int k; + U d; + + xa0 = a->x; + xa = xa0 + a->wds; + y = *--xa; +#ifdef DEBUG + if (!y) Bug("zero y in b2d"); +#endif + k = hi0bits(y); + *e = 32 - k; + if (k < Ebits) { + word0(&d) = Exp_1 | y >> (Ebits - k); + w = xa > xa0 ? *--xa : 0; + word1(&d) = y << ((32-Ebits) + k) | w >> (Ebits - k); + goto ret_d; + } + z = xa > xa0 ? *--xa : 0; + if (k -= Ebits) { + word0(&d) = Exp_1 | y << k | z >> (32 - k); + y = xa > xa0 ? *--xa : 0; + word1(&d) = z << k | y >> (32 - k); + } + else { + word0(&d) = Exp_1 | y; + word1(&d) = z; + } + ret_d: + return dval(&d); +} + +/* Convert a scaled double to a Bigint plus an exponent. Similar to d2b, + except that it accepts the scale parameter used in _Py_dg_strtod (which + should be either 0 or 2*P), and the normalization for the return value is + different (see below). On input, d should be finite and nonnegative, and d + / 2**scale should be exactly representable as an IEEE 754 double. + + Returns a Bigint b and an integer e such that + + dval(d) / 2**scale = b * 2**e. + + Unlike d2b, b is not necessarily odd: b and e are normalized so + that either 2**(P-1) <= b < 2**P and e >= Etiny, or b < 2**P + and e == Etiny. This applies equally to an input of 0.0: in that + case the return values are b = 0 and e = Etiny. + + The above normalization ensures that for all possible inputs d, + 2**e gives ulp(d/2**scale). + + Returns NULL on failure. +*/ + +static Bigint * +sd2b(U *d, int scale, int *e) +{ + Bigint *b; + + b = Balloc(1); + if (b == NULL) + return NULL; + + /* First construct b and e assuming that scale == 0. */ + b->wds = 2; + b->x[0] = word1(d); + b->x[1] = word0(d) & Frac_mask; + *e = Etiny - 1 + (int)((word0(d) & Exp_mask) >> Exp_shift); + if (*e < Etiny) + *e = Etiny; + else + b->x[1] |= Exp_msk1; + + /* Now adjust for scale, provided that b != 0. */ + if (scale && (b->x[0] || b->x[1])) { + *e -= scale; + if (*e < Etiny) { + scale = Etiny - *e; + *e = Etiny; + /* We can't shift more than P-1 bits without shifting out a 1. */ + assert(0 < scale && scale <= P - 1); + if (scale >= 32) { + /* The bits shifted out should all be zero. */ + assert(b->x[0] == 0); + b->x[0] = b->x[1]; + b->x[1] = 0; + scale -= 32; + } + if (scale) { + /* The bits shifted out should all be zero. */ + assert(b->x[0] << (32 - scale) == 0); + b->x[0] = (b->x[0] >> scale) | (b->x[1] << (32 - scale)); + b->x[1] >>= scale; + } + } + } + /* Ensure b is normalized. */ + if (!b->x[1]) + b->wds = 1; + + return b; +} + +/* Convert a double to a Bigint plus an exponent. Return NULL on failure. + + Given a finite nonzero double d, return an odd Bigint b and exponent *e + such that fabs(d) = b * 2**e. On return, *bbits gives the number of + significant bits of b; that is, 2**(*bbits-1) <= b < 2**(*bbits). + + If d is zero, then b == 0, *e == -1010, *bbits = 0. + */ + +static Bigint * +d2b(U *d, int *e, int *bits) +{ + Bigint *b; + int de, k; + ULong *x, y, z; + int i; + + b = Balloc(1); + if (b == NULL) + return NULL; + x = b->x; + + z = word0(d) & Frac_mask; + word0(d) &= 0x7fffffff; /* clear sign bit, which we ignore */ + if ((de = (int)(word0(d) >> Exp_shift))) + z |= Exp_msk1; + if ((y = word1(d))) { + if ((k = lo0bits(&y))) { + x[0] = y | z << (32 - k); + z >>= k; + } + else + x[0] = y; + i = + b->wds = (x[1] = z) ? 2 : 1; + } + else { + k = lo0bits(&z); + x[0] = z; + i = + b->wds = 1; + k += 32; + } + if (de) { + *e = de - Bias - (P-1) + k; + *bits = P - k; + } + else { + *e = de - Bias - (P-1) + 1 + k; + *bits = 32*i - hi0bits(x[i-1]); + } + return b; +} + +/* Compute the ratio of two Bigints, as a double. The result may have an + error of up to 2.5 ulps. */ + +static double +ratio(Bigint *a, Bigint *b) +{ + U da, db; + int k, ka, kb; + + dval(&da) = b2d(a, &ka); + dval(&db) = b2d(b, &kb); + k = ka - kb + 32*(a->wds - b->wds); + if (k > 0) + word0(&da) += k*Exp_msk1; + else { + k = -k; + word0(&db) += k*Exp_msk1; + } + return dval(&da) / dval(&db); +} + +static const double +tens[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, + 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, + 1e20, 1e21, 1e22 +}; + +static const double +bigtens[] = { 1e16, 1e32, 1e64, 1e128, 1e256 }; +static const double tinytens[] = { 1e-16, 1e-32, 1e-64, 1e-128, + 9007199254740992.*9007199254740992.e-256 + /* = 2^106 * 1e-256 */ +}; +/* The factor of 2^53 in tinytens[4] helps us avoid setting the underflow */ +/* flag unnecessarily. It leads to a song and dance at the end of strtod. */ +#define Scale_Bit 0x10 +#define n_bigtens 5 + +#define ULbits 32 +#define kshift 5 +#define kmask 31 + + +static int +dshift(Bigint *b, int p2) +{ + int rv = hi0bits(b->x[b->wds-1]) - 4; + if (p2 > 0) + rv -= p2; + return rv & kmask; +} + +/* special case of Bigint division. The quotient is always in the range 0 <= + quotient < 10, and on entry the divisor S is normalized so that its top 4 + bits (28--31) are zero and bit 27 is set. */ + +static int +quorem(Bigint *b, Bigint *S) +{ + int n; + ULong *bx, *bxe, q, *sx, *sxe; +#ifdef ULLong + ULLong borrow, carry, y, ys; +#else + ULong borrow, carry, y, ys; + ULong si, z, zs; +#endif + + n = S->wds; +#ifdef DEBUG + /*debug*/ if (b->wds > n) + /*debug*/ Bug("oversize b in quorem"); +#endif + if (b->wds < n) + return 0; + sx = S->x; + sxe = sx + --n; + bx = b->x; + bxe = bx + n; + q = *bxe / (*sxe + 1); /* ensure q <= true quotient */ +#ifdef DEBUG + /*debug*/ if (q > 9) + /*debug*/ Bug("oversized quotient in quorem"); +#endif + if (q) { + borrow = 0; + carry = 0; + do { +#ifdef ULLong + ys = *sx++ * (ULLong)q + carry; + carry = ys >> 32; + y = *bx - (ys & FFFFFFFF) - borrow; + borrow = y >> 32 & (ULong)1; + *bx++ = (ULong)(y & FFFFFFFF); +#else + si = *sx++; + ys = (si & 0xffff) * q + carry; + zs = (si >> 16) * q + (ys >> 16); + carry = zs >> 16; + y = (*bx & 0xffff) - (ys & 0xffff) - borrow; + borrow = (y & 0x10000) >> 16; + z = (*bx >> 16) - (zs & 0xffff) - borrow; + borrow = (z & 0x10000) >> 16; + Storeinc(bx, z, y); +#endif + } + while(sx <= sxe); + if (!*bxe) { + bx = b->x; + while(--bxe > bx && !*bxe) + --n; + b->wds = n; + } + } + if (cmp(b, S) >= 0) { + q++; + borrow = 0; + carry = 0; + bx = b->x; + sx = S->x; + do { +#ifdef ULLong + ys = *sx++ + carry; + carry = ys >> 32; + y = *bx - (ys & FFFFFFFF) - borrow; + borrow = y >> 32 & (ULong)1; + *bx++ = (ULong)(y & FFFFFFFF); +#else + si = *sx++; + ys = (si & 0xffff) + carry; + zs = (si >> 16) + (ys >> 16); + carry = zs >> 16; + y = (*bx & 0xffff) - (ys & 0xffff) - borrow; + borrow = (y & 0x10000) >> 16; + z = (*bx >> 16) - (zs & 0xffff) - borrow; + borrow = (z & 0x10000) >> 16; + Storeinc(bx, z, y); +#endif + } + while(sx <= sxe); + bx = b->x; + bxe = bx + n; + if (!*bxe) { + while(--bxe > bx && !*bxe) + --n; + b->wds = n; + } + } + return q; +} + +/* sulp(x) is a version of ulp(x) that takes bc.scale into account. + + Assuming that x is finite and nonnegative (positive zero is fine + here) and x / 2^bc.scale is exactly representable as a double, + sulp(x) is equivalent to 2^bc.scale * ulp(x / 2^bc.scale). */ + +static double +sulp(U *x, BCinfo *bc) +{ + U u; + + if (bc->scale && 2*P + 1 > (int)((word0(x) & Exp_mask) >> Exp_shift)) { + /* rv/2^bc->scale is subnormal */ + word0(&u) = (P+2)*Exp_msk1; + word1(&u) = 0; + return u.d; + } + else { + assert(word0(x) || word1(x)); /* x != 0.0 */ + return ulp(x); + } +} + +/* The bigcomp function handles some hard cases for strtod, for inputs + with more than STRTOD_DIGLIM digits. It's called once an initial + estimate for the double corresponding to the input string has + already been obtained by the code in _Py_dg_strtod. + + The bigcomp function is only called after _Py_dg_strtod has found a + double value rv such that either rv or rv + 1ulp represents the + correctly rounded value corresponding to the original string. It + determines which of these two values is the correct one by + computing the decimal digits of rv + 0.5ulp and comparing them with + the corresponding digits of s0. + + In the following, write dv for the absolute value of the number represented + by the input string. + + Inputs: + + s0 points to the first significant digit of the input string. + + rv is a (possibly scaled) estimate for the closest double value to the + value represented by the original input to _Py_dg_strtod. If + bc->scale is nonzero, then rv/2^(bc->scale) is the approximation to + the input value. + + bc is a struct containing information gathered during the parsing and + estimation steps of _Py_dg_strtod. Description of fields follows: + + bc->e0 gives the exponent of the input value, such that dv = (integer + given by the bd->nd digits of s0) * 10**e0 + + bc->nd gives the total number of significant digits of s0. It will + be at least 1. + + bc->nd0 gives the number of significant digits of s0 before the + decimal separator. If there's no decimal separator, bc->nd0 == + bc->nd. + + bc->scale is the value used to scale rv to avoid doing arithmetic with + subnormal values. It's either 0 or 2*P (=106). + + Outputs: + + On successful exit, rv/2^(bc->scale) is the closest double to dv. + + Returns 0 on success, -1 on failure (e.g., due to a failed malloc call). */ + +static int +bigcomp(U *rv, const char *s0, BCinfo *bc) +{ + Bigint *b, *d; + int b2, d2, dd, i, nd, nd0, odd, p2, p5; + + nd = bc->nd; + nd0 = bc->nd0; + p5 = nd + bc->e0; + b = sd2b(rv, bc->scale, &p2); + if (b == NULL) + return -1; + + /* record whether the lsb of rv/2^(bc->scale) is odd: in the exact halfway + case, this is used for round to even. */ + odd = b->x[0] & 1; + + /* left shift b by 1 bit and or a 1 into the least significant bit; + this gives us b * 2**p2 = rv/2^(bc->scale) + 0.5 ulp. */ + b = lshift(b, 1); + if (b == NULL) + return -1; + b->x[0] |= 1; + p2--; + + p2 -= p5; + d = i2b(1); + if (d == NULL) { + Bfree(b); + return -1; + } + /* Arrange for convenient computation of quotients: + * shift left if necessary so divisor has 4 leading 0 bits. + */ + if (p5 > 0) { + d = pow5mult(d, p5); + if (d == NULL) { + Bfree(b); + return -1; + } + } + else if (p5 < 0) { + b = pow5mult(b, -p5); + if (b == NULL) { + Bfree(d); + return -1; + } + } + if (p2 > 0) { + b2 = p2; + d2 = 0; + } + else { + b2 = 0; + d2 = -p2; + } + i = dshift(d, d2); + if ((b2 += i) > 0) { + b = lshift(b, b2); + if (b == NULL) { + Bfree(d); + return -1; + } + } + if ((d2 += i) > 0) { + d = lshift(d, d2); + if (d == NULL) { + Bfree(b); + return -1; + } + } + + /* Compare s0 with b/d: set dd to -1, 0, or 1 according as s0 < b/d, s0 == + * b/d, or s0 > b/d. Here the digits of s0 are thought of as representing + * a number in the range [0.1, 1). */ + if (cmp(b, d) >= 0) + /* b/d >= 1 */ + dd = -1; + else { + i = 0; + for(;;) { + b = multadd(b, 10, 0); + if (b == NULL) { + Bfree(d); + return -1; + } + dd = s0[i < nd0 ? i : i+1] - '0' - quorem(b, d); + i++; + + if (dd) + break; + if (!b->x[0] && b->wds == 1) { + /* b/d == 0 */ + dd = i < nd; + break; + } + if (!(i < nd)) { + /* b/d != 0, but digits of s0 exhausted */ + dd = -1; + break; + } + } + } + Bfree(b); + Bfree(d); + if (dd > 0 || (dd == 0 && odd)) + dval(rv) += sulp(rv, bc); + return 0; +} + +double +_Py_dg_strtod(const char *s00, char **se) +{ + int bb2, bb5, bbe, bd2, bd5, bs2, c, dsign, e, e1, error; + int esign, i, j, k, lz, nd, nd0, odd, sign; + const char *s, *s0, *s1; + double aadj, aadj1; + U aadj2, adj, rv, rv0; + ULong y, z, abs_exp; + Long L; + BCinfo bc; + Bigint *bb, *bb1, *bd, *bd0, *bs, *delta; + + dval(&rv) = 0.; + + /* Start parsing. */ + c = *(s = s00); + + /* Parse optional sign, if present. */ + sign = 0; + switch (c) { + case '-': + sign = 1; + /* no break */ + case '+': + c = *++s; + } + + /* Skip leading zeros: lz is true iff there were leading zeros. */ + s1 = s; + while (c == '0') + c = *++s; + lz = s != s1; + + /* Point s0 at the first nonzero digit (if any). nd0 will be the position + of the point relative to s0. nd will be the total number of digits + ignoring leading zeros. */ + s0 = s1 = s; + while ('0' <= c && c <= '9') + c = *++s; + nd0 = nd = s - s1; + + /* Parse decimal point and following digits. */ + if (c == '.') { + c = *++s; + if (!nd) { + s1 = s; + while (c == '0') + c = *++s; + lz = lz || s != s1; + nd0 -= s - s1; + s0 = s; + } + s1 = s; + while ('0' <= c && c <= '9') + c = *++s; + nd += s - s1; + } + + /* Now lz is true if and only if there were leading zero digits, and nd + gives the total number of digits ignoring leading zeros. A valid input + must have at least one digit. */ + if (!nd && !lz) { + if (se) + *se = (char *)s00; + goto parse_error; + } + + /* Parse exponent. */ + e = 0; + if (c == 'e' || c == 'E') { + s00 = s; + c = *++s; + + /* Exponent sign. */ + esign = 0; + switch (c) { + case '-': + esign = 1; + /* no break */ + case '+': + c = *++s; + } + + /* Skip zeros. lz is true iff there are leading zeros. */ + s1 = s; + while (c == '0') + c = *++s; + lz = s != s1; + + /* Get absolute value of the exponent. */ + s1 = s; + abs_exp = 0; + while ('0' <= c && c <= '9') { + abs_exp = 10*abs_exp + (c - '0'); + c = *++s; + } + + /* abs_exp will be correct modulo 2**32. But 10**9 < 2**32, so if + there are at most 9 significant exponent digits then overflow is + impossible. */ + if (s - s1 > 9 || abs_exp > MAX_ABS_EXP) + e = (int)MAX_ABS_EXP; + else + e = (int)abs_exp; + if (esign) + e = -e; + + /* A valid exponent must have at least one digit. */ + if (s == s1 && !lz) + s = s00; + } + + /* Adjust exponent to take into account position of the point. */ + e -= nd - nd0; + if (nd0 <= 0) + nd0 = nd; + + /* Finished parsing. Set se to indicate how far we parsed */ + if (se) + *se = (char *)s; + + /* If all digits were zero, exit with return value +-0.0. Otherwise, + strip trailing zeros: scan back until we hit a nonzero digit. */ + if (!nd) + goto ret; + for (i = nd; i > 0; ) { + --i; + if (s0[i < nd0 ? i : i+1] != '0') { + ++i; + break; + } + } + e += nd - i; + nd = i; + if (nd0 > nd) + nd0 = nd; + + /* Summary of parsing results. After parsing, and dealing with zero + * inputs, we have values s0, nd0, nd, e, sign, where: + * + * - s0 points to the first significant digit of the input string + * + * - nd is the total number of significant digits (here, and + * below, 'significant digits' means the set of digits of the + * significand of the input that remain after ignoring leading + * and trailing zeros). + * + * - nd0 indicates the position of the decimal point, if present; it + * satisfies 1 <= nd0 <= nd. The nd significant digits are in + * s0[0:nd0] and s0[nd0+1:nd+1] using the usual Python half-open slice + * notation. (If nd0 < nd, then s0[nd0] contains a '.' character; if + * nd0 == nd, then s0[nd0] could be any non-digit character.) + * + * - e is the adjusted exponent: the absolute value of the number + * represented by the original input string is n * 10**e, where + * n is the integer represented by the concatenation of + * s0[0:nd0] and s0[nd0+1:nd+1] + * + * - sign gives the sign of the input: 1 for negative, 0 for positive + * + * - the first and last significant digits are nonzero + */ + + /* put first DBL_DIG+1 digits into integer y and z. + * + * - y contains the value represented by the first min(9, nd) + * significant digits + * + * - if nd > 9, z contains the value represented by significant digits + * with indices in [9, min(16, nd)). So y * 10**(min(16, nd) - 9) + z + * gives the value represented by the first min(16, nd) sig. digits. + */ + + bc.e0 = e1 = e; + y = z = 0; + for (i = 0; i < nd; i++) { + if (i < 9) + y = 10*y + s0[i < nd0 ? i : i+1] - '0'; + else if (i < DBL_DIG+1) + z = 10*z + s0[i < nd0 ? i : i+1] - '0'; + else + break; + } + + k = nd < DBL_DIG + 1 ? nd : DBL_DIG + 1; + dval(&rv) = y; + if (k > 9) { + dval(&rv) = tens[k - 9] * dval(&rv) + z; + } + bd0 = 0; + if (nd <= DBL_DIG + && Flt_Rounds == 1 + ) { + if (!e) + goto ret; + if (e > 0) { + if (e <= Ten_pmax) { + dval(&rv) *= tens[e]; + goto ret; + } + i = DBL_DIG - nd; + if (e <= Ten_pmax + i) { + /* A fancier test would sometimes let us do + * this for larger i values. + */ + e -= i; + dval(&rv) *= tens[i]; + dval(&rv) *= tens[e]; + goto ret; + } + } + else if (e >= -Ten_pmax) { + dval(&rv) /= tens[-e]; + goto ret; + } + } + e1 += nd - k; + + bc.scale = 0; + + /* Get starting approximation = rv * 10**e1 */ + + if (e1 > 0) { + if ((i = e1 & 15)) + dval(&rv) *= tens[i]; + if (e1 &= ~15) { + if (e1 > DBL_MAX_10_EXP) + goto ovfl; + e1 >>= 4; + for(j = 0; e1 > 1; j++, e1 >>= 1) + if (e1 & 1) + dval(&rv) *= bigtens[j]; + /* The last multiplication could overflow. */ + word0(&rv) -= P*Exp_msk1; + dval(&rv) *= bigtens[j]; + if ((z = word0(&rv) & Exp_mask) + > Exp_msk1*(DBL_MAX_EXP+Bias-P)) + goto ovfl; + if (z > Exp_msk1*(DBL_MAX_EXP+Bias-1-P)) { + /* set to largest number */ + /* (Can't trust DBL_MAX) */ + word0(&rv) = Big0; + word1(&rv) = Big1; + } + else + word0(&rv) += P*Exp_msk1; + } + } + else if (e1 < 0) { + /* The input decimal value lies in [10**e1, 10**(e1+16)). + + If e1 <= -512, underflow immediately. + If e1 <= -256, set bc.scale to 2*P. + + So for input value < 1e-256, bc.scale is always set; + for input value >= 1e-240, bc.scale is never set. + For input values in [1e-256, 1e-240), bc.scale may or may + not be set. */ + + e1 = -e1; + if ((i = e1 & 15)) + dval(&rv) /= tens[i]; + if (e1 >>= 4) { + if (e1 >= 1 << n_bigtens) + goto undfl; + if (e1 & Scale_Bit) + bc.scale = 2*P; + for(j = 0; e1 > 0; j++, e1 >>= 1) + if (e1 & 1) + dval(&rv) *= tinytens[j]; + if (bc.scale && (j = 2*P + 1 - ((word0(&rv) & Exp_mask) + >> Exp_shift)) > 0) { + /* scaled rv is denormal; clear j low bits */ + if (j >= 32) { + word1(&rv) = 0; + if (j >= 53) + word0(&rv) = (P+2)*Exp_msk1; + else + word0(&rv) &= 0xffffffff << (j-32); + } + else + word1(&rv) &= 0xffffffff << j; + } + if (!dval(&rv)) + goto undfl; + } + } + + /* Now the hard part -- adjusting rv to the correct value.*/ + + /* Put digits into bd: true value = bd * 10^e */ + + bc.nd = nd; + bc.nd0 = nd0; /* Only needed if nd > STRTOD_DIGLIM, but done here */ + /* to silence an erroneous warning about bc.nd0 */ + /* possibly not being initialized. */ + if (nd > STRTOD_DIGLIM) { + /* ASSERT(STRTOD_DIGLIM >= 18); 18 == one more than the */ + /* minimum number of decimal digits to distinguish double values */ + /* in IEEE arithmetic. */ + + /* Truncate input to 18 significant digits, then discard any trailing + zeros on the result by updating nd, nd0, e and y suitably. (There's + no need to update z; it's not reused beyond this point.) */ + for (i = 18; i > 0; ) { + /* scan back until we hit a nonzero digit. significant digit 'i' + is s0[i] if i < nd0, s0[i+1] if i >= nd0. */ + --i; + if (s0[i < nd0 ? i : i+1] != '0') { + ++i; + break; + } + } + e += nd - i; + nd = i; + if (nd0 > nd) + nd0 = nd; + if (nd < 9) { /* must recompute y */ + y = 0; + for(i = 0; i < nd0; ++i) + y = 10*y + s0[i] - '0'; + for(; i < nd; ++i) + y = 10*y + s0[i+1] - '0'; + } + } + bd0 = s2b(s0, nd0, nd, y); + if (bd0 == NULL) + goto failed_malloc; + + /* Notation for the comments below. Write: + + - dv for the absolute value of the number represented by the original + decimal input string. + + - if we've truncated dv, write tdv for the truncated value. + Otherwise, set tdv == dv. + + - srv for the quantity rv/2^bc.scale; so srv is the current binary + approximation to tdv (and dv). It should be exactly representable + in an IEEE 754 double. + */ + + for(;;) { + + /* This is the main correction loop for _Py_dg_strtod. + + We've got a decimal value tdv, and a floating-point approximation + srv=rv/2^bc.scale to tdv. The aim is to determine whether srv is + close enough (i.e., within 0.5 ulps) to tdv, and to compute a new + approximation if not. + + To determine whether srv is close enough to tdv, compute integers + bd, bb and bs proportional to tdv, srv and 0.5 ulp(srv) + respectively, and then use integer arithmetic to determine whether + |tdv - srv| is less than, equal to, or greater than 0.5 ulp(srv). + */ + + bd = Balloc(bd0->k); + if (bd == NULL) { + Bfree(bd0); + goto failed_malloc; + } + Bcopy(bd, bd0); + bb = sd2b(&rv, bc.scale, &bbe); /* srv = bb * 2^bbe */ + if (bb == NULL) { + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + /* Record whether lsb of bb is odd, in case we need this + for the round-to-even step later. */ + odd = bb->x[0] & 1; + + /* tdv = bd * 10**e; srv = bb * 2**bbe */ + bs = i2b(1); + if (bs == NULL) { + Bfree(bb); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + + if (e >= 0) { + bb2 = bb5 = 0; + bd2 = bd5 = e; + } + else { + bb2 = bb5 = -e; + bd2 = bd5 = 0; + } + if (bbe >= 0) + bb2 += bbe; + else + bd2 -= bbe; + bs2 = bb2; + bb2++; + bd2++; + + /* At this stage bd5 - bb5 == e == bd2 - bb2 + bbe, bb2 - bs2 == 1, + and bs == 1, so: + + tdv == bd * 10**e = bd * 2**(bbe - bb2 + bd2) * 5**(bd5 - bb5) + srv == bb * 2**bbe = bb * 2**(bbe - bb2 + bb2) + 0.5 ulp(srv) == 2**(bbe-1) = bs * 2**(bbe - bb2 + bs2) + + It follows that: + + M * tdv = bd * 2**bd2 * 5**bd5 + M * srv = bb * 2**bb2 * 5**bb5 + M * 0.5 ulp(srv) = bs * 2**bs2 * 5**bb5 + + for some constant M. (Actually, M == 2**(bb2 - bbe) * 5**bb5, but + this fact is not needed below.) + */ + + /* Remove factor of 2**i, where i = min(bb2, bd2, bs2). */ + i = bb2 < bd2 ? bb2 : bd2; + if (i > bs2) + i = bs2; + if (i > 0) { + bb2 -= i; + bd2 -= i; + bs2 -= i; + } + + /* Scale bb, bd, bs by the appropriate powers of 2 and 5. */ + if (bb5 > 0) { + bs = pow5mult(bs, bb5); + if (bs == NULL) { + Bfree(bb); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + bb1 = mult(bs, bb); + Bfree(bb); + bb = bb1; + if (bb == NULL) { + Bfree(bs); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + } + if (bb2 > 0) { + bb = lshift(bb, bb2); + if (bb == NULL) { + Bfree(bs); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + } + if (bd5 > 0) { + bd = pow5mult(bd, bd5); + if (bd == NULL) { + Bfree(bb); + Bfree(bs); + Bfree(bd0); + goto failed_malloc; + } + } + if (bd2 > 0) { + bd = lshift(bd, bd2); + if (bd == NULL) { + Bfree(bb); + Bfree(bs); + Bfree(bd0); + goto failed_malloc; + } + } + if (bs2 > 0) { + bs = lshift(bs, bs2); + if (bs == NULL) { + Bfree(bb); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + } + + /* Now bd, bb and bs are scaled versions of tdv, srv and 0.5 ulp(srv), + respectively. Compute the difference |tdv - srv|, and compare + with 0.5 ulp(srv). */ + + delta = diff(bb, bd); + if (delta == NULL) { + Bfree(bb); + Bfree(bs); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + dsign = delta->sign; + delta->sign = 0; + i = cmp(delta, bs); + if (bc.nd > nd && i <= 0) { + if (dsign) + break; /* Must use bigcomp(). */ + + /* Here rv overestimates the truncated decimal value by at most + 0.5 ulp(rv). Hence rv either overestimates the true decimal + value by <= 0.5 ulp(rv), or underestimates it by some small + amount (< 0.1 ulp(rv)); either way, rv is within 0.5 ulps of + the true decimal value, so it's possible to exit. + + Exception: if scaled rv is a normal exact power of 2, but not + DBL_MIN, then rv - 0.5 ulp(rv) takes us all the way down to the + next double, so the correctly rounded result is either rv - 0.5 + ulp(rv) or rv; in this case, use bigcomp to distinguish. */ + + if (!word1(&rv) && !(word0(&rv) & Bndry_mask)) { + /* rv can't be 0, since it's an overestimate for some + nonzero value. So rv is a normal power of 2. */ + j = (int)(word0(&rv) & Exp_mask) >> Exp_shift; + /* rv / 2^bc.scale = 2^(j - 1023 - bc.scale); use bigcomp if + rv / 2^bc.scale >= 2^-1021. */ + if (j - bc.scale >= 2) { + dval(&rv) -= 0.5 * sulp(&rv, &bc); + break; /* Use bigcomp. */ + } + } + + { + bc.nd = nd; + i = -1; /* Discarded digits make delta smaller. */ + } + } + + if (i < 0) { + /* Error is less than half an ulp -- check for + * special case of mantissa a power of two. + */ + if (dsign || word1(&rv) || word0(&rv) & Bndry_mask + || (word0(&rv) & Exp_mask) <= (2*P+1)*Exp_msk1 + ) { + break; + } + if (!delta->x[0] && delta->wds <= 1) { + /* exact result */ + break; + } + delta = lshift(delta,Log2P); + if (delta == NULL) { + Bfree(bb); + Bfree(bs); + Bfree(bd); + Bfree(bd0); + goto failed_malloc; + } + if (cmp(delta, bs) > 0) + goto drop_down; + break; + } + if (i == 0) { + /* exactly half-way between */ + if (dsign) { + if ((word0(&rv) & Bndry_mask1) == Bndry_mask1 + && word1(&rv) == ( + (bc.scale && + (y = word0(&rv) & Exp_mask) <= 2*P*Exp_msk1) ? + (0xffffffff & (0xffffffff << (2*P+1-(y>>Exp_shift)))) : + 0xffffffff)) { + /*boundary case -- increment exponent*/ + word0(&rv) = (word0(&rv) & Exp_mask) + + Exp_msk1 + ; + word1(&rv) = 0; + dsign = 0; + break; + } + } + else if (!(word0(&rv) & Bndry_mask) && !word1(&rv)) { + drop_down: + /* boundary case -- decrement exponent */ + if (bc.scale) { + L = word0(&rv) & Exp_mask; + if (L <= (2*P+1)*Exp_msk1) { + if (L > (P+2)*Exp_msk1) + /* round even ==> */ + /* accept rv */ + break; + /* rv = smallest denormal */ + if (bc.nd > nd) + break; + goto undfl; + } + } + L = (word0(&rv) & Exp_mask) - Exp_msk1; + word0(&rv) = L | Bndry_mask1; + word1(&rv) = 0xffffffff; + break; + } + if (!odd) + break; + if (dsign) + dval(&rv) += sulp(&rv, &bc); + else { + dval(&rv) -= sulp(&rv, &bc); + if (!dval(&rv)) { + if (bc.nd >nd) + break; + goto undfl; + } + } + dsign = 1 - dsign; + break; + } + if ((aadj = ratio(delta, bs)) <= 2.) { + if (dsign) + aadj = aadj1 = 1.; + else if (word1(&rv) || word0(&rv) & Bndry_mask) { + if (word1(&rv) == Tiny1 && !word0(&rv)) { + if (bc.nd >nd) + break; + goto undfl; + } + aadj = 1.; + aadj1 = -1.; + } + else { + /* special case -- power of FLT_RADIX to be */ + /* rounded down... */ + + if (aadj < 2./FLT_RADIX) + aadj = 1./FLT_RADIX; + else + aadj *= 0.5; + aadj1 = -aadj; + } + } + else { + aadj *= 0.5; + aadj1 = dsign ? aadj : -aadj; + if (Flt_Rounds == 0) + aadj1 += 0.5; + } + y = word0(&rv) & Exp_mask; + + /* Check for overflow */ + + if (y == Exp_msk1*(DBL_MAX_EXP+Bias-1)) { + dval(&rv0) = dval(&rv); + word0(&rv) -= P*Exp_msk1; + adj.d = aadj1 * ulp(&rv); + dval(&rv) += adj.d; + if ((word0(&rv) & Exp_mask) >= + Exp_msk1*(DBL_MAX_EXP+Bias-P)) { + if (word0(&rv0) == Big0 && word1(&rv0) == Big1) { + Bfree(bb); + Bfree(bd); + Bfree(bs); + Bfree(bd0); + Bfree(delta); + goto ovfl; + } + word0(&rv) = Big0; + word1(&rv) = Big1; + goto cont; + } + else + word0(&rv) += P*Exp_msk1; + } + else { + if (bc.scale && y <= 2*P*Exp_msk1) { + if (aadj <= 0x7fffffff) { + if ((z = (ULong)aadj) <= 0) + z = 1; + aadj = z; + aadj1 = dsign ? aadj : -aadj; + } + dval(&aadj2) = aadj1; + word0(&aadj2) += (2*P+1)*Exp_msk1 - y; + aadj1 = dval(&aadj2); + } + adj.d = aadj1 * ulp(&rv); + dval(&rv) += adj.d; + } + z = word0(&rv) & Exp_mask; + if (bc.nd == nd) { + if (!bc.scale) + if (y == z) { + /* Can we stop now? */ + L = (Long)aadj; + aadj -= L; + /* The tolerances below are conservative. */ + if (dsign || word1(&rv) || word0(&rv) & Bndry_mask) { + if (aadj < .4999999 || aadj > .5000001) + break; + } + else if (aadj < .4999999/FLT_RADIX) + break; + } + } + cont: + Bfree(bb); + Bfree(bd); + Bfree(bs); + Bfree(delta); + } + Bfree(bb); + Bfree(bd); + Bfree(bs); + Bfree(bd0); + Bfree(delta); + if (bc.nd > nd) { + error = bigcomp(&rv, s0, &bc); + if (error) + goto failed_malloc; + } + + if (bc.scale) { + word0(&rv0) = Exp_1 - 2*P*Exp_msk1; + word1(&rv0) = 0; + dval(&rv) *= dval(&rv0); + } + + ret: + return sign ? -dval(&rv) : dval(&rv); + + parse_error: + return 0.0; + + failed_malloc: + errno = ENOMEM; + return -1.0; + + undfl: + return sign ? -0.0 : 0.0; + + ovfl: + errno = ERANGE; + /* Can't trust HUGE_VAL */ + word0(&rv) = Exp_mask; + word1(&rv) = 0; + return sign ? -dval(&rv) : dval(&rv); + +} + +static char * +rv_alloc(int i) +{ + int j, k, *r; + + j = sizeof(ULong); + for(k = 0; + sizeof(Bigint) - sizeof(ULong) - sizeof(int) + j <= (unsigned)i; + j <<= 1) + k++; + r = (int*)Balloc(k); + if (r == NULL) + return NULL; + *r = k; + return (char *)(r+1); +} + +static char * +nrv_alloc(char *s, char **rve, int n) +{ + char *rv, *t; + + rv = rv_alloc(n); + if (rv == NULL) + return NULL; + t = rv; + while((*t = *s++)) t++; + if (rve) + *rve = t; + return rv; +} + +/* freedtoa(s) must be used to free values s returned by dtoa + * when MULTIPLE_THREADS is #defined. It should be used in all cases, + * but for consistency with earlier versions of dtoa, it is optional + * when MULTIPLE_THREADS is not defined. + */ + +void +_Py_dg_freedtoa(char *s) +{ + Bigint *b = (Bigint *)((int *)s - 1); + b->maxwds = 1 << (b->k = *(int*)b); + Bfree(b); +} + +/* dtoa for IEEE arithmetic (dmg): convert double to ASCII string. + * + * Inspired by "How to Print Floating-Point Numbers Accurately" by + * Guy L. Steele, Jr. and Jon L. White [Proc. ACM SIGPLAN '90, pp. 112-126]. + * + * Modifications: + * 1. Rather than iterating, we use a simple numeric overestimate + * to determine k = floor(log10(d)). We scale relevant + * quantities using O(log2(k)) rather than O(k) multiplications. + * 2. For some modes > 2 (corresponding to ecvt and fcvt), we don't + * try to generate digits strictly left to right. Instead, we + * compute with fewer bits and propagate the carry if necessary + * when rounding the final digit up. This is often faster. + * 3. Under the assumption that input will be rounded nearest, + * mode 0 renders 1e23 as 1e23 rather than 9.999999999999999e22. + * That is, we allow equality in stopping tests when the + * round-nearest rule will give the same floating-point value + * as would satisfaction of the stopping test with strict + * inequality. + * 4. We remove common factors of powers of 2 from relevant + * quantities. + * 5. When converting floating-point integers less than 1e16, + * we use floating-point arithmetic rather than resorting + * to multiple-precision integers. + * 6. When asked to produce fewer than 15 digits, we first try + * to get by with floating-point arithmetic; we resort to + * multiple-precision integer arithmetic only if we cannot + * guarantee that the floating-point calculation has given + * the correctly rounded result. For k requested digits and + * "uniformly" distributed input, the probability is + * something like 10^(k-15) that we must resort to the Long + * calculation. + */ + +/* Additional notes (METD): (1) returns NULL on failure. (2) to avoid memory + leakage, a successful call to _Py_dg_dtoa should always be matched by a + call to _Py_dg_freedtoa. */ + +char * +_Py_dg_dtoa(double dd, int mode, int ndigits, + int *decpt, int *sign, char **rve) +{ + /* Arguments ndigits, decpt, sign are similar to those + of ecvt and fcvt; trailing zeros are suppressed from + the returned string. If not null, *rve is set to point + to the end of the return value. If d is +-Infinity or NaN, + then *decpt is set to 9999. + + mode: + 0 ==> shortest string that yields d when read in + and rounded to nearest. + 1 ==> like 0, but with Steele & White stopping rule; + e.g. with IEEE P754 arithmetic , mode 0 gives + 1e23 whereas mode 1 gives 9.999999999999999e22. + 2 ==> max(1,ndigits) significant digits. This gives a + return value similar to that of ecvt, except + that trailing zeros are suppressed. + 3 ==> through ndigits past the decimal point. This + gives a return value similar to that from fcvt, + except that trailing zeros are suppressed, and + ndigits can be negative. + 4,5 ==> similar to 2 and 3, respectively, but (in + round-nearest mode) with the tests of mode 0 to + possibly return a shorter string that rounds to d. + With IEEE arithmetic and compilation with + -DHonor_FLT_ROUNDS, modes 4 and 5 behave the same + as modes 2 and 3 when FLT_ROUNDS != 1. + 6-9 ==> Debugging modes similar to mode - 4: don't try + fast floating-point estimate (if applicable). + + Values of mode other than 0-9 are treated as mode 0. + + Sufficient space is allocated to the return value + to hold the suppressed trailing zeros. + */ + + int bbits, b2, b5, be, dig, i, ieps, ilim, ilim0, ilim1, + j, j1, k, k0, k_check, leftright, m2, m5, s2, s5, + spec_case, try_quick; + Long L; + int denorm; + ULong x; + Bigint *b, *b1, *delta, *mlo, *mhi, *S; + U d2, eps, u; + double ds; + char *s, *s0; + + /* set pointers to NULL, to silence gcc compiler warnings and make + cleanup easier on error */ + mlo = mhi = S = 0; + s0 = 0; + + u.d = dd; + if (word0(&u) & Sign_bit) { + /* set sign for everything, including 0's and NaNs */ + *sign = 1; + word0(&u) &= ~Sign_bit; /* clear sign bit */ + } + else + *sign = 0; + + /* quick return for Infinities, NaNs and zeros */ + if ((word0(&u) & Exp_mask) == Exp_mask) + { + /* Infinity or NaN */ + *decpt = 9999; + if (!word1(&u) && !(word0(&u) & 0xfffff)) + return nrv_alloc("Infinity", rve, 8); + return nrv_alloc("NaN", rve, 3); + } + if (!dval(&u)) { + *decpt = 1; + return nrv_alloc("0", rve, 1); + } + + /* compute k = floor(log10(d)). The computation may leave k + one too large, but should never leave k too small. */ + b = d2b(&u, &be, &bbits); + if (b == NULL) + goto failed_malloc; + if ((i = (int)(word0(&u) >> Exp_shift1 & (Exp_mask>>Exp_shift1)))) { + dval(&d2) = dval(&u); + word0(&d2) &= Frac_mask1; + word0(&d2) |= Exp_11; + + /* log(x) ~=~ log(1.5) + (x-1.5)/1.5 + * log10(x) = log(x) / log(10) + * ~=~ log(1.5)/log(10) + (x-1.5)/(1.5*log(10)) + * log10(d) = (i-Bias)*log(2)/log(10) + log10(d2) + * + * This suggests computing an approximation k to log10(d) by + * + * k = (i - Bias)*0.301029995663981 + * + ( (d2-1.5)*0.289529654602168 + 0.176091259055681 ); + * + * We want k to be too large rather than too small. + * The error in the first-order Taylor series approximation + * is in our favor, so we just round up the constant enough + * to compensate for any error in the multiplication of + * (i - Bias) by 0.301029995663981; since |i - Bias| <= 1077, + * and 1077 * 0.30103 * 2^-52 ~=~ 7.2e-14, + * adding 1e-13 to the constant term more than suffices. + * Hence we adjust the constant term to 0.1760912590558. + * (We could get a more accurate k by invoking log10, + * but this is probably not worthwhile.) + */ + + i -= Bias; + denorm = 0; + } + else { + /* d is denormalized */ + + i = bbits + be + (Bias + (P-1) - 1); + x = i > 32 ? word0(&u) << (64 - i) | word1(&u) >> (i - 32) + : word1(&u) << (32 - i); + dval(&d2) = x; + word0(&d2) -= 31*Exp_msk1; /* adjust exponent */ + i -= (Bias + (P-1) - 1) + 1; + denorm = 1; + } + ds = (dval(&d2)-1.5)*0.289529654602168 + 0.1760912590558 + + i*0.301029995663981; + k = (int)ds; + if (ds < 0. && ds != k) + k--; /* want k = floor(ds) */ + k_check = 1; + if (k >= 0 && k <= Ten_pmax) { + if (dval(&u) < tens[k]) + k--; + k_check = 0; + } + j = bbits - i - 1; + if (j >= 0) { + b2 = 0; + s2 = j; + } + else { + b2 = -j; + s2 = 0; + } + if (k >= 0) { + b5 = 0; + s5 = k; + s2 += k; + } + else { + b2 -= k; + b5 = -k; + s5 = 0; + } + if (mode < 0 || mode > 9) + mode = 0; + + try_quick = 1; + + if (mode > 5) { + mode -= 4; + try_quick = 0; + } + leftright = 1; + ilim = ilim1 = -1; /* Values for cases 0 and 1; done here to */ + /* silence erroneous "gcc -Wall" warning. */ + switch(mode) { + case 0: + case 1: + i = 18; + ndigits = 0; + break; + case 2: + leftright = 0; + /* no break */ + case 4: + if (ndigits <= 0) + ndigits = 1; + ilim = ilim1 = i = ndigits; + break; + case 3: + leftright = 0; + /* no break */ + case 5: + i = ndigits + k + 1; + ilim = i; + ilim1 = i - 1; + if (i <= 0) + i = 1; + } + s0 = rv_alloc(i); + if (s0 == NULL) + goto failed_malloc; + s = s0; + + + if (ilim >= 0 && ilim <= Quick_max && try_quick) { + + /* Try to get by with floating-point arithmetic. */ + + i = 0; + dval(&d2) = dval(&u); + k0 = k; + ilim0 = ilim; + ieps = 2; /* conservative */ + if (k > 0) { + ds = tens[k&0xf]; + j = k >> 4; + if (j & Bletch) { + /* prevent overflows */ + j &= Bletch - 1; + dval(&u) /= bigtens[n_bigtens-1]; + ieps++; + } + for(; j; j >>= 1, i++) + if (j & 1) { + ieps++; + ds *= bigtens[i]; + } + dval(&u) /= ds; + } + else if ((j1 = -k)) { + dval(&u) *= tens[j1 & 0xf]; + for(j = j1 >> 4; j; j >>= 1, i++) + if (j & 1) { + ieps++; + dval(&u) *= bigtens[i]; + } + } + if (k_check && dval(&u) < 1. && ilim > 0) { + if (ilim1 <= 0) + goto fast_failed; + ilim = ilim1; + k--; + dval(&u) *= 10.; + ieps++; + } + dval(&eps) = ieps*dval(&u) + 7.; + word0(&eps) -= (P-1)*Exp_msk1; + if (ilim == 0) { + S = mhi = 0; + dval(&u) -= 5.; + if (dval(&u) > dval(&eps)) + goto one_digit; + if (dval(&u) < -dval(&eps)) + goto no_digits; + goto fast_failed; + } + if (leftright) { + /* Use Steele & White method of only + * generating digits needed. + */ + dval(&eps) = 0.5/tens[ilim-1] - dval(&eps); + for(i = 0;;) { + L = (Long)dval(&u); + dval(&u) -= L; + *s++ = '0' + (int)L; + if (dval(&u) < dval(&eps)) + goto ret1; + if (1. - dval(&u) < dval(&eps)) + goto bump_up; + if (++i >= ilim) + break; + dval(&eps) *= 10.; + dval(&u) *= 10.; + } + } + else { + /* Generate ilim digits, then fix them up. */ + dval(&eps) *= tens[ilim-1]; + for(i = 1;; i++, dval(&u) *= 10.) { + L = (Long)(dval(&u)); + if (!(dval(&u) -= L)) + ilim = i; + *s++ = '0' + (int)L; + if (i == ilim) { + if (dval(&u) > 0.5 + dval(&eps)) + goto bump_up; + else if (dval(&u) < 0.5 - dval(&eps)) { + while(*--s == '0'); + s++; + goto ret1; + } + break; + } + } + } + fast_failed: + s = s0; + dval(&u) = dval(&d2); + k = k0; + ilim = ilim0; + } + + /* Do we have a "small" integer? */ + + if (be >= 0 && k <= Int_max) { + /* Yes. */ + ds = tens[k]; + if (ndigits < 0 && ilim <= 0) { + S = mhi = 0; + if (ilim < 0 || dval(&u) <= 5*ds) + goto no_digits; + goto one_digit; + } + for(i = 1;; i++, dval(&u) *= 10.) { + L = (Long)(dval(&u) / ds); + dval(&u) -= L*ds; + *s++ = '0' + (int)L; + if (!dval(&u)) { + break; + } + if (i == ilim) { + dval(&u) += dval(&u); + if (dval(&u) > ds || (dval(&u) == ds && L & 1)) { + bump_up: + while(*--s == '9') + if (s == s0) { + k++; + *s = '0'; + break; + } + ++*s++; + } + break; + } + } + goto ret1; + } + + m2 = b2; + m5 = b5; + if (leftright) { + i = + denorm ? be + (Bias + (P-1) - 1 + 1) : + 1 + P - bbits; + b2 += i; + s2 += i; + mhi = i2b(1); + if (mhi == NULL) + goto failed_malloc; + } + if (m2 > 0 && s2 > 0) { + i = m2 < s2 ? m2 : s2; + b2 -= i; + m2 -= i; + s2 -= i; + } + if (b5 > 0) { + if (leftright) { + if (m5 > 0) { + mhi = pow5mult(mhi, m5); + if (mhi == NULL) + goto failed_malloc; + b1 = mult(mhi, b); + Bfree(b); + b = b1; + if (b == NULL) + goto failed_malloc; + } + if ((j = b5 - m5)) { + b = pow5mult(b, j); + if (b == NULL) + goto failed_malloc; + } + } + else { + b = pow5mult(b, b5); + if (b == NULL) + goto failed_malloc; + } + } + S = i2b(1); + if (S == NULL) + goto failed_malloc; + if (s5 > 0) { + S = pow5mult(S, s5); + if (S == NULL) + goto failed_malloc; + } + + /* Check for special case that d is a normalized power of 2. */ + + spec_case = 0; + if ((mode < 2 || leftright) + ) { + if (!word1(&u) && !(word0(&u) & Bndry_mask) + && word0(&u) & (Exp_mask & ~Exp_msk1) + ) { + /* The special case */ + b2 += Log2P; + s2 += Log2P; + spec_case = 1; + } + } + + /* Arrange for convenient computation of quotients: + * shift left if necessary so divisor has 4 leading 0 bits. + * + * Perhaps we should just compute leading 28 bits of S once + * and for all and pass them and a shift to quorem, so it + * can do shifts and ors to compute the numerator for q. + */ +#define iInc 28 + i = dshift(S, s2); + b2 += i; + m2 += i; + s2 += i; + if (b2 > 0) { + b = lshift(b, b2); + if (b == NULL) + goto failed_malloc; + } + if (s2 > 0) { + S = lshift(S, s2); + if (S == NULL) + goto failed_malloc; + } + if (k_check) { + if (cmp(b,S) < 0) { + k--; + b = multadd(b, 10, 0); /* we botched the k estimate */ + if (b == NULL) + goto failed_malloc; + if (leftright) { + mhi = multadd(mhi, 10, 0); + if (mhi == NULL) + goto failed_malloc; + } + ilim = ilim1; + } + } + if (ilim <= 0 && (mode == 3 || mode == 5)) { + if (ilim < 0) { + /* no digits, fcvt style */ + no_digits: + k = -1 - ndigits; + goto ret; + } + else { + S = multadd(S, 5, 0); + if (S == NULL) + goto failed_malloc; + if (cmp(b, S) <= 0) + goto no_digits; + } + one_digit: + *s++ = '1'; + k++; + goto ret; + } + if (leftright) { + if (m2 > 0) { + mhi = lshift(mhi, m2); + if (mhi == NULL) + goto failed_malloc; + } + + /* Compute mlo -- check for special case + * that d is a normalized power of 2. + */ + + mlo = mhi; + if (spec_case) { + mhi = Balloc(mhi->k); + if (mhi == NULL) + goto failed_malloc; + Bcopy(mhi, mlo); + mhi = lshift(mhi, Log2P); + if (mhi == NULL) + goto failed_malloc; + } + + for(i = 1;;i++) { + dig = quorem(b,S) + '0'; + /* Do we yet have the shortest decimal string + * that will round to d? + */ + j = cmp(b, mlo); + delta = diff(S, mhi); + if (delta == NULL) + goto failed_malloc; + j1 = delta->sign ? 1 : cmp(b, delta); + Bfree(delta); + if (j1 == 0 && mode != 1 && !(word1(&u) & 1) + ) { + if (dig == '9') + goto round_9_up; + if (j > 0) + dig++; + *s++ = dig; + goto ret; + } + if (j < 0 || (j == 0 && mode != 1 + && !(word1(&u) & 1) + )) { + if (!b->x[0] && b->wds <= 1) { + goto accept_dig; + } + if (j1 > 0) { + b = lshift(b, 1); + if (b == NULL) + goto failed_malloc; + j1 = cmp(b, S); + if ((j1 > 0 || (j1 == 0 && dig & 1)) + && dig++ == '9') + goto round_9_up; + } + accept_dig: + *s++ = dig; + goto ret; + } + if (j1 > 0) { + if (dig == '9') { /* possible if i == 1 */ + round_9_up: + *s++ = '9'; + goto roundoff; + } + *s++ = dig + 1; + goto ret; + } + *s++ = dig; + if (i == ilim) + break; + b = multadd(b, 10, 0); + if (b == NULL) + goto failed_malloc; + if (mlo == mhi) { + mlo = mhi = multadd(mhi, 10, 0); + if (mlo == NULL) + goto failed_malloc; + } + else { + mlo = multadd(mlo, 10, 0); + if (mlo == NULL) + goto failed_malloc; + mhi = multadd(mhi, 10, 0); + if (mhi == NULL) + goto failed_malloc; + } + } + } + else + for(i = 1;; i++) { + *s++ = dig = quorem(b,S) + '0'; + if (!b->x[0] && b->wds <= 1) { + goto ret; + } + if (i >= ilim) + break; + b = multadd(b, 10, 0); + if (b == NULL) + goto failed_malloc; + } + + /* Round off last digit */ + + b = lshift(b, 1); + if (b == NULL) + goto failed_malloc; + j = cmp(b, S); + if (j > 0 || (j == 0 && dig & 1)) { + roundoff: + while(*--s == '9') + if (s == s0) { + k++; + *s++ = '1'; + goto ret; + } + ++*s++; + } + else { + while(*--s == '0'); + s++; + } + ret: + Bfree(S); + if (mhi) { + if (mlo && mlo != mhi) + Bfree(mlo); + Bfree(mhi); + } + ret1: + Bfree(b); + *s = 0; + *decpt = k + 1; + if (rve) + *rve = s; + return s0; + failed_malloc: + if (S) + Bfree(S); + if (mlo && mlo != mhi) + Bfree(mlo); + if (mhi) + Bfree(mhi); + if (b) + Bfree(b); + if (s0) + _Py_dg_freedtoa(s0); + return NULL; +} +#ifdef __cplusplus +} +#endif + +#endif /* PY_NO_SHORT_FLOAT_REPR */ Added: pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py Tue Dec 7 23:52:57 2010 @@ -0,0 +1,80 @@ +from __future__ import with_statement +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.tool.autopath import pypydir +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib.rstring import StringBuilder +import py + +includes = [] +libraries = [] + +cdir = py.path.local(pypydir) / 'translator' / 'c' +files = [cdir / 'src' / 'dtoa.c'] +include_dirs = [cdir] + +eci = ExternalCompilationInfo( + include_dirs = include_dirs, + libraries = libraries, + separate_module_files = files, + separate_module_sources = [''' + #include + #include + #define WITH_PYMALLOC + #include "src/obmalloc.c" + '''], + export_symbols = ['_Py_dg_strtod', + '_Py_dg_dtoa', + '_Py_dg_freedtoa', + ], +) + +dg_strtod = rffi.llexternal( + '_Py_dg_strtod', [rffi.CCHARP, rffi.CCHARPP], rffi.DOUBLE, + compilation_info=eci) + +dg_dtoa = rffi.llexternal( + '_Py_dg_dtoa', [rffi.DOUBLE, rffi.INT, rffi.INT, + rffi.INTP, rffi.INTP, rffi.CCHARPP], rffi.CCHARP, + compilation_info=eci) + +dg_freedtoa = rffi.llexternal( + '_Py_dg_freedtoa', [rffi.CCHARP], lltype.Void, + compilation_info=eci) + +def strtod(input): + with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as end_ptr: + with rffi.scoped_str2charp(input) as ll_input: + result = dg_strtod(ll_input, end_ptr) + if end_ptr[0] and ord(end_ptr[0][0]): + offset = (rffi.cast(rffi.LONG, end_ptr[0]) - + rffi.cast(rffi.LONG, ll_input)) + raise ValueError("invalid input at position %d" % (offset,)) + return result + +def dtoa(value): + mode = 2 + precision = 3 + builder = StringBuilder(20) + with lltype.scoped_alloc(rffi.INTP.TO, 1) as decpt_ptr: + with lltype.scoped_alloc(rffi.INTP.TO, 1) as sign_ptr: + with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as end_ptr: + output_ptr = dg_dtoa(value, mode, precision, + decpt_ptr, sign_ptr, end_ptr) + buflen = (rffi.cast(rffi.LONG, end_ptr[0]) - + rffi.cast(rffi.LONG, output_ptr)) + builder.append(rffi.charpsize2str(output_ptr, decpt_ptr[0])) + builder.append('.') + ptr = rffi.ptradd(output_ptr, decpt_ptr[0]) + buflen -= decpt_ptr[0] + builder.append(rffi.charpsize2str(ptr, buflen)) + dg_freedtoa(output_ptr) + return builder.build() + +def test_strtod(): + assert strtod("12345") == 12345.0 + assert strtod("1.1") == 1.1 + raises(ValueError, strtod, "123A") + +def test_dtoa(): + assert dtoa(3.47) == "3.47" + assert dtoa(1.1) == "1.1" From afa at codespeak.net Tue Dec 7 23:57:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 7 Dec 2010 23:57:10 +0100 (CET) Subject: [pypy-svn] r79881 - pypy/branch/fast-forward/pypy/translator/c/test Message-ID: <20101207225710.DA903282BAD@codespeak.net> Author: afa Date: Tue Dec 7 23:57:09 2010 New Revision: 79881 Modified: pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py Log: Fix test on Linux Modified: pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py Tue Dec 7 23:57:09 2010 @@ -64,7 +64,8 @@ rffi.cast(rffi.LONG, output_ptr)) builder.append(rffi.charpsize2str(output_ptr, decpt_ptr[0])) builder.append('.') - ptr = rffi.ptradd(output_ptr, decpt_ptr[0]) + ptr = rffi.ptradd(output_ptr, + rffi.cast(lltype.Signed, decpt_ptr[0])) buflen -= decpt_ptr[0] builder.append(rffi.charpsize2str(ptr, buflen)) dg_freedtoa(output_ptr) From agaynor at codespeak.net Wed Dec 8 05:46:12 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 8 Dec 2010 05:46:12 +0100 (CET) Subject: [pypy-svn] r79882 - pypy/branch/fast-forward/lib-python Message-ID: <20101208044612.EA56D282BAD@codespeak.net> Author: agaynor Date: Wed Dec 8 05:46:10 2010 New Revision: 79882 Modified: pypy/branch/fast-forward/lib-python/TODO (contents, props changed) Log: list.index doesn't take None for an index, also fixed the EOL on this file. Modified: pypy/branch/fast-forward/lib-python/TODO ============================================================================== --- pypy/branch/fast-forward/lib-python/TODO (original) +++ pypy/branch/fast-forward/lib-python/TODO Wed Dec 8 05:46:10 2010 @@ -1,86 +1,86 @@ -TODO list for 2.7.0 -=================== - -You can find the results of the most recent buildbot run at: -http://buildbot.pypy.org/summary?branch=branch/fast-forward - - -Probably easy tasks -------------------- - -- Missing builtin: bytearray (possibly reuse module.__pypy__.bytebuffer) - -- Missing complex.__trunc__ - -- New complex syntax:: - - assert complex("(1+2j)") == (1+2j) - assert complex("(1-2j)") == (1-2j) - assert complex("1e500") == complex(INF, 0.0) - -- Mark some tests as "implementation specific":: - - @test_support.cpython_only - -- (list|unicode|bytearray).(index|find) should accept None as indices - -- Fix fcntl.fcntl(fd, fcntl.F_NOTIFY, fcntl.DN_MULTISHOT) on 32bit platform. - -- missing functions in itertools: combinations, product, compress... - compress, product DONE - -- in test_os.py, fix posix.setregid(-1, -1), posix.setreuid(-1, -1). This - proably requires to use the git_t typedef instead of rffi.INT. - -- missing posix.(confstr|pathconf|fpathconf)(_names)? - -- remove code duplication: bit_length() and _count_bits() in rlib/rbigint.py, - objspace/std/longobject.py and objspace/std/longtype.py. - - -Medium tasks ------------- - -- Ast objects should be picklable, see in pypy/module/_ast/test/test_ast.py: - test_pickle() - -- missing builtin: memoryview DONE (but not very much tested) - -- Dictionary views: dict.viewkeys(), dict.viewvalues(), dict.viewitems() - -- add 'unicode' in ObjSpace.MethodTable + probably a default implementation that - falls back to space.str(). - -- socket module has a couple of changes (including AF_TIPC packet range) - -Longer tasks ------------- - -- Finish the _io module. - -- Finish _multiprocessing - -- many features are missing from the _ssl module - -- Enable -3 option to run test_py3kwarn. - -- "Shorter float representation": copy dtoa.c from CPython and use it to - format/parse floats. Enable this with a translation option. - -- Fix usage of __cmp__ in subclasses:: - - class badint(int): - def __cmp__(self, other): - raise RuntimeError - raises(RuntimeError, cmp, 0, badint(1)) - -More difficult issues ---------------------- - -- In socket.py, """The implementation currently relies on reference counting to - close the underlying socket object.""" - -- Implement an after-fork hook (See PyOS_AfterFork in CPython) to clear thread - state in a forked interpreter. - Then unskip test_3_join_in_forked_from_thread() in test_threading.py. - +TODO list for 2.7.0 +=================== + +You can find the results of the most recent buildbot run at: +http://buildbot.pypy.org/summary?branch=branch/fast-forward + + +Probably easy tasks +------------------- + +- Missing builtin: bytearray (possibly reuse module.__pypy__.bytebuffer) + +- Missing complex.__trunc__ + +- New complex syntax:: + + assert complex("(1+2j)") == (1+2j) + assert complex("(1-2j)") == (1-2j) + assert complex("1e500") == complex(INF, 0.0) + +- Mark some tests as "implementation specific":: + + @test_support.cpython_only + +- (unicode|bytearray).(index|find) should accept None as indices + +- Fix fcntl.fcntl(fd, fcntl.F_NOTIFY, fcntl.DN_MULTISHOT) on 32bit platform. + +- missing functions in itertools: combinations, product, compress... + compress, product DONE + +- in test_os.py, fix posix.setregid(-1, -1), posix.setreuid(-1, -1). This + proably requires to use the git_t typedef instead of rffi.INT. + +- missing posix.(confstr|pathconf|fpathconf)(_names)? + +- remove code duplication: bit_length() and _count_bits() in rlib/rbigint.py, + objspace/std/longobject.py and objspace/std/longtype.py. + + +Medium tasks +------------ + +- Ast objects should be picklable, see in pypy/module/_ast/test/test_ast.py: + test_pickle() + +- missing builtin: memoryview DONE (but not very much tested) + +- Dictionary views: dict.viewkeys(), dict.viewvalues(), dict.viewitems() + +- add 'unicode' in ObjSpace.MethodTable + probably a default implementation that + falls back to space.str(). + +- socket module has a couple of changes (including AF_TIPC packet range) + +Longer tasks +------------ + +- Finish the _io module. + +- Finish _multiprocessing + +- many features are missing from the _ssl module + +- Enable -3 option to run test_py3kwarn. + +- "Shorter float representation": copy dtoa.c from CPython and use it to + format/parse floats. Enable this with a translation option. + +- Fix usage of __cmp__ in subclasses:: + + class badint(int): + def __cmp__(self, other): + raise RuntimeError + raises(RuntimeError, cmp, 0, badint(1)) + +More difficult issues +--------------------- + +- In socket.py, """The implementation currently relies on reference counting to + close the underlying socket object.""" + +- Implement an after-fork hook (See PyOS_AfterFork in CPython) to clear thread + state in a forked interpreter. + Then unskip test_3_join_in_forked_from_thread() in test_threading.py. + From agaynor at codespeak.net Wed Dec 8 05:50:03 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 8 Dec 2010 05:50:03 +0100 (CET) Subject: [pypy-svn] r79883 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20101208045003.1B754282BAD@codespeak.net> Author: agaynor Date: Wed Dec 8 05:50:00 2010 New Revision: 79883 Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py Log: Fix the other half of the various index taking methods on strings. Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/stringobject.py Wed Dec 8 05:50:00 2010 @@ -387,6 +387,8 @@ def _convert_idx_params(space, w_self, w_sub, w_start, w_end, upper_bound=False): self = w_self._value sub = w_sub._value + if space.is_w(w_start, space.w_None): + w_start = space.wrap(0) if space.is_w(w_end, space.w_None): w_end = space.len(w_self) if upper_bound: Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py Wed Dec 8 05:50:00 2010 @@ -414,6 +414,7 @@ assert 'abcdefghiabc'.index('abc', 1) == 9 assert 'abcdefghiabc'.index('def', -4*maxint, 4*maxint) == 3 assert 'abcdefgh'.index('def', 2, None) == 3 + assert 'abcdefgh'.index('def', None, None) == 3 raises(ValueError, 'abcdefghiabc'.index, 'hib') raises(ValueError, 'abcdefghiab'.index, 'abc', 1) raises(ValueError, 'abcdefghi'.index, 'ghi', 8) From agaynor at codespeak.net Wed Dec 8 06:02:58 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 8 Dec 2010 06:02:58 +0100 (CET) Subject: [pypy-svn] r79884 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20101208050258.BCEFC282BAD@codespeak.net> Author: agaynor Date: Wed Dec 8 06:02:55 2010 New Revision: 79884 Modified: pypy/branch/fast-forward/pypy/objspace/std/complextype.py pypy/branch/fast-forward/pypy/objspace/std/test/test_complexobject.py Log: Started implementing enhanced complex constructor parsing. Modified: pypy/branch/fast-forward/pypy/objspace/std/complextype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/complextype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/complextype.py Wed Dec 8 06:02:55 2010 @@ -32,6 +32,10 @@ # ignore whitespace while i < slen and s[i] == ' ': i += 1 + + if s[i] == '(' and s[slen-1] == ')': + i += 1 + slen -= 1 # extract first number realstart = i Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_complexobject.py Wed Dec 8 06:02:55 2010 @@ -36,6 +36,9 @@ test_cparse('3L+3j', '3L', '3') test_cparse('3j', '0.0', '3') test_cparse('.e+5', '.e+5', '0.0') + test_cparse('(1+2j)', '1', '2') + test_cparse('(1-6j)', '1', '-6') + def test_pow(self): assert cobj._pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) From agaynor at codespeak.net Wed Dec 8 07:29:51 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 8 Dec 2010 07:29:51 +0100 (CET) Subject: [pypy-svn] r79885 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20101208062951.942AB282BAD@codespeak.net> Author: agaynor Date: Wed Dec 8 07:29:45 2010 New Revision: 79885 Modified: pypy/branch/fast-forward/pypy/objspace/std/ropeobject.py pypy/branch/fast-forward/pypy/objspace/std/stringobject.py Log: Fixed ropeobject. Modified: pypy/branch/fast-forward/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/ropeobject.py Wed Dec 8 07:29:45 2010 @@ -15,7 +15,7 @@ from pypy.rlib import rope from pypy.objspace.std.stringobject import mod__String_ANY as mod__Rope_ANY,\ - _upper, _lower + _upper, _lower class W_RopeObject(W_Object): from pypy.objspace.std.stringtype import str_typedef as typedef @@ -61,7 +61,7 @@ registerimplementation(W_RopeIterObject) -def _is_generic(space, w_self, fun): +def _is_generic(space, w_self, fun): l = w_self._node.length() if l == 0: return space.w_False @@ -93,7 +93,7 @@ """Return True if all cased characters in S are uppercase and there is at least one cased character in S, False otherwise.""" l = w_self._node.length() - + if l == 0: return space.w_False cased = False @@ -110,7 +110,7 @@ """Return True if all cased characters in S are lowercase and there is at least one cased character in S, False otherwise.""" l = w_self._node.length() - + if l == 0: return space.w_False cased = False @@ -178,7 +178,7 @@ def str_swapcase__Rope(space, w_self): return _local_transform(w_self._node, _swapcase) - + def str_capitalize__Rope(space, w_self): node = w_self._node length = node.length() @@ -203,7 +203,7 @@ return W_RopeObject.EMPTY return W_RopeObject(rope.rope_from_charlist(buffer)) - + def str_title__Rope(space, w_self): node = w_self._node @@ -306,7 +306,7 @@ if len(fillchar) != 1: raise OperationError(space.w_TypeError, space.wrap("rjust() argument 2 must be a single character")) - + d = u_arg - selfnode.length() if d > 0: fillchar = fillchar[0] # annotator hint: it's a single character @@ -317,7 +317,7 @@ return W_RopeObject(resultnode) else: return W_RopeObject(selfnode) - + def str_ljust__Rope_ANY_ANY(space, w_self, w_arg, w_fillchar): u_arg = space.int_w(w_arg) selfnode = w_self._node @@ -325,7 +325,7 @@ if len(fillchar) != 1: raise OperationError(space.w_TypeError, space.wrap("rjust() argument 2 must be a single character")) - + d = u_arg - selfnode.length() if d > 0: fillchar = fillchar[0] # annotator hint: it's a single character @@ -341,6 +341,10 @@ self = w_self._node sub = w_sub._node + if space.is_w(w_start, space.w_None): + w_start = space.wrap(0) + if space.is_w(w_end, space.w_None): + w_end = space.len(w_self) if upper_bound: start = slicetype.adapt_bound(space, self.length(), w_start) end = slicetype.adapt_bound(space, self.length(), w_end) @@ -475,7 +479,7 @@ def str_strip__Rope_None(space, w_self, w_chars): return W_RopeObject(rope.strip(w_self._node, left=True, right=True)) - + def str_rstrip__Rope_Rope(space, w_self, w_chars): return W_RopeObject(rope.strip(w_self._node, False, True, _contains, w_chars._node.flatten_string())) @@ -511,7 +515,7 @@ else: return w_self.create_if_subclassed() -def str_count__Rope_Rope_ANY_ANY(space, w_self, w_arg, w_start, w_end): +def str_count__Rope_Rope_ANY_ANY(space, w_self, w_arg, w_start, w_end): selfnode = w_self._node length = selfnode.length() argnode = w_arg._node @@ -546,18 +550,18 @@ w_u = space.call_function(space.w_unicode, w_self) return space.call_method(w_u, "endswith", w_suffixes, w_start, w_end) - suffix = rope_w(space, w_suffix) + suffix = rope_w(space, w_suffix) if rope.endswith(self, suffix, start, end): return space.w_True return space.w_False - + def str_startswith__Rope_Rope_ANY_ANY(space, w_self, w_prefix, w_start, w_end): (self, prefix, start, end) = _convert_idx_params(space, w_self, w_prefix, w_start, w_end, True) return space.newbool(rope.startswith(self, prefix, start, end)) - + def str_startswith__Rope_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end): (self, _, start, end) = _convert_idx_params(space, w_self, W_RopeObject.EMPTY, w_start, w_end, True) @@ -570,7 +574,7 @@ if rope.startswith(self, prefix, start, end): return space.w_True return space.w_False - + def _tabindent(node, tabsize): "calculates distance after the token to the next tabstop" @@ -590,22 +594,22 @@ if char == ord("\n") or char == ord("\r"): break distance += 1 - + #the same like distance = len(u_token) - (offset + 1) distance = (tabsize - distance) % tabsize if distance == 0: return tabsize - return distance - - -def str_expandtabs__Rope_ANY(space, w_self, w_tabsize): + return distance + + +def str_expandtabs__Rope_ANY(space, w_self, w_tabsize): node = w_self._node length = node.length() if length == 0: return W_RopeObject.EMPTY tabsize = space.int_w(w_tabsize) - + splitted = rope.split(node, rope.LiteralStringNode.PREBUILT[ord("\t")]) last = splitted[0] expanded = [last] @@ -617,9 +621,9 @@ try: return W_RopeObject(rope.rebalance(expanded)) except OverflowError: - raise OperationError(space.w_OverflowError, space.wrap('new string is too long')) - - + raise OperationError(space.w_OverflowError, space.wrap('new string is too long')) + + def str_splitlines__Rope_ANY(space, w_self, w_keepends): keepends = bool(space.int_w(w_keepends)) # truth value, but type checked node = w_self._node @@ -819,10 +823,10 @@ def str_translate__Rope_ANY_ANY(space, w_string, w_table, w_deletechars=''): """charfilter - unicode handling is not implemented - - Return a copy of the string where all characters occurring - in the optional argument deletechars are removed, and the - remaining characters have been mapped through the given translation table, + + Return a copy of the string where all characters occurring + in the optional argument deletechars are removed, and the + remaining characters have been mapped through the given translation table, which must be a string of length 256""" # XXX CPython accepts buffers, too, not sure what we should do @@ -876,15 +880,15 @@ def next__RopeIter(space, w_ropeiter): if w_ropeiter.node is None: - raise OperationError(space.w_StopIteration, space.w_None) + raise OperationError(space.w_StopIteration, space.w_None) try: char = w_ropeiter.item_iter.nextchar() w_item = wrapchar(space, char) except StopIteration: w_ropeiter.node = None w_ropeiter.char_iter = None - raise OperationError(space.w_StopIteration, space.w_None) - w_ropeiter.index += 1 + raise OperationError(space.w_StopIteration, space.w_None) + w_ropeiter.index += 1 return w_item # XXX __length_hint__() Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/stringobject.py Wed Dec 8 07:29:45 2010 @@ -387,6 +387,7 @@ def _convert_idx_params(space, w_self, w_sub, w_start, w_end, upper_bound=False): self = w_self._value sub = w_sub._value + if space.is_w(w_start, space.w_None): w_start = space.wrap(0) if space.is_w(w_end, space.w_None): From david at codespeak.net Wed Dec 8 09:55:21 2010 From: david at codespeak.net (david at codespeak.net) Date: Wed, 8 Dec 2010 09:55:21 +0100 (CET) Subject: [pypy-svn] r79886 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101208085521.C52EA282BAD@codespeak.net> Author: david Date: Wed Dec 8 09:55:19 2010 New Revision: 79886 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Log: Refactor location encodings Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Wed Dec 8 09:55:19 2010 @@ -22,6 +22,29 @@ sandboxsafe=True, _nowrapper=True) class AssemblerARM(ResOpAssembler): + """ + Encoding for locations in memory + types: + \xEE = REF + \xEF = INT + location: + \xFC = stack location + \xFD = imm location + emtpy = reg location + \xFE = Empty loc + + \xFF = END_OF_LOCS + """ + REF_TYPE = '\xEE' + INT_TYPE = '\xEF' + + STACK_LOC = '\xFC' + IMM_LOC = '\xFD' + # REG_LOC is empty + EMPTY_LOC = '\xFE' + + END_OF_LOCS = '\xFF' + def __init__(self, cpu, failargs_limit=1000): self.cpu = cpu @@ -99,36 +122,36 @@ i += 1 fail_index += 1 res = enc[i] - if res == '\xFF': + if res == self.END_OF_LOCS: break - if res == '\xFE': + if res == self.EMPTY_LOC: continue group = res i += 1 res = enc[i] - if res == '\xFD': - assert group == '\xEF' + if res == self.IMM_LOC: + assert group == self.INT_TYPE # imm value value = self.decode32(enc, i+1) i += 4 - elif res == '\xFC': # stack location + elif res == self.STACK_LOC: stack_loc = self.decode32(enc, i+1) value = self.decode32(stack, frame_depth - stack_loc*WORD) i += 4 - else: # an int in a reg location + else: # REG_LOC reg = ord(enc[i]) value = self.decode32(regs, reg*WORD) - if group == '\xEF': # INT + if group == self.INT_TYPE: self.fail_boxes_int.setitem(fail_index, value) - elif group == '\xEE': # REF + elif group == self.REF_TYPE: self.fail_boxes_ptr.setitem(fail_index, rffi.cast(llmemory.GCREF, value)) else: assert 0, 'unknown type' - assert enc[i] == '\xFF' + assert enc[i] == self.END_OF_LOCS descr = self.decode32(enc, i+1) self.fail_boxes_count = fail_index return descr @@ -138,23 +161,23 @@ j = 0 for i in range(len(inputargs)): res = enc[j] - if res == '\xFF': + if res == self.END_OF_LOCS: assert 0, 'reached end of encoded area' - # XXX decode imm and and stack locs and REFs - while res == '\xFE': + while res == self.EMPTY_LOC: j += 1 res = enc[j] - assert res == '\xEF' # only int location for now + assert res in [self.INT_TYPE, self.REF_TYPE], 'location type is not supported' j += 1 res = enc[j] - if res == '\xFD': # imm location - assert 0, 'fail' - elif res == '\xFC': # stack location + if res == self.IMM_LOC: + # XXX decode imm if necessary + assert 0, 'Imm Locations are not supported' + elif res == self.STACK_LOC: stack_loc = self.decode32(enc, j+1) loc = regalloc.frame_manager.frame_pos(stack_loc, INT) j += 4 - else: # reg location + else: # REG_LOC loc = r.all_regs[ord(res)] j += 1 locs.append(loc) @@ -191,16 +214,6 @@ self.gen_func_epilog() def _gen_path_to_exit_path(self, op, args, regalloc, fcond=c.AL): - """ - types: - \xEE = REF - \xEF = INT - location: - \xFC = stack location - \xFD = imm location - emtpy = reg location - \xFE = Empty arg - """ descr = op.getdescr() if op.getopnum() != rop.FINISH: @@ -217,10 +230,10 @@ if args[i]: loc = regalloc.loc(args[i]) if args[i].type == INT: - mem[j] = '\xEF' + mem[j] = self.INT_TYPE j += 1 elif args[i].type == REF: - mem[j] = '\xEE' + mem[j] = self.REF_TYPE j += 1 else: assert 0, 'unknown type' @@ -230,15 +243,15 @@ j += 1 elif loc.is_imm(): assert args[i].type == INT - mem[j] = '\xFD' + mem[j] = self.IMM_LOC self.encode32(mem, j+1, loc.getint()) j += 5 else: - mem[j] = '\xFC' + mem[j] = self.STACK_LOC self.encode32(mem, j+1, loc.position) j += 5 else: - mem[j] = '\xFE' + mem[j] = self.EMPTY_LOC j += 1 i += 1 From afa at codespeak.net Wed Dec 8 10:33:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 8 Dec 2010 10:33:46 +0100 (CET) Subject: [pypy-svn] r79887 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101208093346.64111282BAD@codespeak.net> Author: afa Date: Wed Dec 8 10:33:43 2010 New Revision: 79887 Added: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/list_tests.py - copied, changed from r79885, pypy/branch/fast-forward/lib-python/2.7.0/test/list_tests.py Log: pypy doesn't respect sys.getrecursionlimit() anymore. Use some large number instead. Copied: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/list_tests.py (from r79885, pypy/branch/fast-forward/lib-python/2.7.0/test/list_tests.py) ============================================================================== --- pypy/branch/fast-forward/lib-python/2.7.0/test/list_tests.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/list_tests.py Wed Dec 8 10:33:43 2010 @@ -45,8 +45,12 @@ self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + if test_support.check_impl_detail(): + depth = sys.getrecursionlimit() + 100 + else: + depth = 1000 * 1000 # should be enough to exhaust the stack l0 = [] - for i in xrange(sys.getrecursionlimit() + 100): + for i in xrange(depth): l0 = [l0] self.assertRaises(RuntimeError, repr, l0) From david at codespeak.net Wed Dec 8 11:21:43 2010 From: david at codespeak.net (david at codespeak.net) Date: Wed, 8 Dec 2010 11:21:43 +0100 (CET) Subject: [pypy-svn] r79888 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101208102143.2FD24282BAD@codespeak.net> Author: david Date: Wed Dec 8 11:21:40 2010 New Revision: 79888 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: Make sure guard_nonnull_class fits in current piece of memory Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Wed Dec 8 11:21:40 2010 @@ -265,8 +265,12 @@ return fcond def emit_op_guard_nonnull_class(self, op, regalloc, fcond): - locs = self._prepare_guard_class(op, regalloc, fcond) offset = self.cpu.vtable_offset + if offset is not None: + self.mc.ensure_can_fit(self.guard_size+3*WORD) + else: + raise NotImplementedError + locs = self._prepare_guard_class(op, regalloc, fcond) self.mc.CMP_ri(locs[0].value, 0) if offset is not None: From arigo at codespeak.net Wed Dec 8 13:38:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 13:38:43 +0100 (CET) Subject: [pypy-svn] r79889 - pypy/trunk/pypy/doc/config Message-ID: <20101208123843.2613E282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 13:38:42 2010 New Revision: 79889 Added: pypy/trunk/pypy/doc/config/translation.jit_ffi.txt Log: Add option. Added: pypy/trunk/pypy/doc/config/translation.jit_ffi.txt ============================================================================== --- (empty file) +++ pypy/trunk/pypy/doc/config/translation.jit_ffi.txt Wed Dec 8 13:38:42 2010 @@ -0,0 +1 @@ +Internal option: enable OptFfiCall in the jit optimizations. From arigo at codespeak.net Wed Dec 8 14:06:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 14:06:17 +0100 (CET) Subject: [pypy-svn] r79890 - in pypy/trunk/pypy: interpreter/pyparser module/exceptions Message-ID: <20101208130617.28816282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 14:06:15 2010 New Revision: 79890 Modified: pypy/trunk/pypy/interpreter/pyparser/error.py pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py pypy/trunk/pypy/module/exceptions/interp_exceptions.py Log: Fix for test_compiler. All we need is to make sure that in case more input is expected because of an opening parenthesis or tripe-quote is not closed so far, then str(SyntaxError()) is different for the different SyntaxErrors we get when adding a blank line after the input. So I made these SyntaxErrors report "lines x-y" instead of just "line x", where "x" is the line where the opening parenthesis was found, and "y" is the last line in the file. Modified: pypy/trunk/pypy/interpreter/pyparser/error.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/error.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/error.py Wed Dec 8 14:06:15 2010 @@ -2,19 +2,22 @@ class SyntaxError(Exception): """Base class for exceptions raised by the parser.""" - def __init__(self, msg, lineno=0, offset=0, text=None, filename=None): + def __init__(self, msg, lineno=0, offset=0, text=None, filename=None, + lastlineno=0): self.msg = msg self.lineno = lineno self.offset = offset self.text = text self.filename = filename + self.lastlineno = lastlineno def wrap_info(self, space): return space.newtuple([space.wrap(self.msg), space.newtuple([space.wrap(self.filename), space.wrap(self.lineno), space.wrap(self.offset), - space.wrap(self.text)])]) + space.wrap(self.text), + space.wrap(self.lastlineno)])]) def __str__(self): return "%s at pos (%d, %d) in %r" % (self.__class__.__name__, @@ -33,8 +36,9 @@ class TokenError(SyntaxError): - def __init__(self, msg, line, lineno, column, tokens): - SyntaxError.__init__(self, msg, lineno, column, line) + def __init__(self, msg, line, lineno, column, tokens, lastlineno=0): + SyntaxError.__init__(self, msg, lineno, column, line, + lastlineno=lastlineno) self.tokens = tokens class TokenIndentationError(IndentationError): Modified: pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/pytokenizer.py Wed Dec 8 14:06:15 2010 @@ -95,7 +95,7 @@ if not line: raise TokenError("EOF while scanning triple-quoted string", strstart[2], strstart[0], strstart[1]+1, - token_list) + token_list, lnum) endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch @@ -148,12 +148,12 @@ else: # continued statement if not line: - start = 0 if parenlev > 0: - lnum, start, line = parenlevstart - start += 1 + lnum1, start1, line1 = parenlevstart + raise TokenError("parenthesis is never closed", line1, + lnum1, start1 + 1, token_list, lnum) raise TokenError("EOF in multi-line statement", line, - lnum, start, token_list) + lnum, 0, token_list) continued = 0 while pos < max: Modified: pypy/trunk/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/trunk/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/trunk/pypy/module/exceptions/interp_exceptions.py Wed Dec 8 14:06:15 2010 @@ -452,6 +452,7 @@ self.w_text = space.w_None self.w_msg = space.wrap('') self.w_print_file_and_line = space.w_None # what's that? + self.w_lastlineno = space.w_None # this is a pypy extension W_BaseException.__init__(self, space) def descr_init(self, space, args_w): @@ -459,11 +460,12 @@ if len(args_w) > 0: self.w_msg = args_w[0] if len(args_w) == 2: - values_w = space.fixedview(args_w[1], 4) - self.w_filename = values_w[0] - self.w_lineno = values_w[1] - self.w_offset = values_w[2] - self.w_text = values_w[3] + values_w = space.fixedview(args_w[1]) + if len(values_w) > 0: self.w_filename = values_w[0] + if len(values_w) > 1: self.w_lineno = values_w[1] + if len(values_w) > 2: self.w_offset = values_w[2] + if len(values_w) > 3: self.w_text = values_w[3] + if len(values_w) > 4: self.w_lastlineno = values_w[4] W_BaseException.descr_init(self, space, args_w) descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] @@ -472,18 +474,24 @@ if type(self.msg) is not str: return str(self.msg) + lineno = None buffer = self.msg have_filename = type(self.filename) is str - have_lineno = type(self.lineno) is int + if type(self.lineno) is int: + if (type(self.lastlineno) is int and + self.lastlineno > self.lineno): + lineno = 'lines %d-%d' % (self.lineno, self.lastlineno) + else: + lineno = 'line %d' % (self.lineno,) if have_filename: import os fname = os.path.basename(self.filename or "???") - if have_lineno: - buffer = "%s (%s, line %ld)" % (self.msg, fname, self.lineno) + if lineno: + buffer = "%s (%s, %s)" % (self.msg, fname, lineno) else: buffer ="%s (%s)" % (self.msg, fname) - elif have_lineno: - buffer = "%s (line %ld)" % (self.msg, self.lineno) + elif lineno: + buffer = "%s (%s)" % (self.msg, lineno) return buffer """) @@ -504,6 +512,7 @@ text = readwrite_attrproperty_w('w_text', W_SyntaxError), print_file_and_line = readwrite_attrproperty_w('w_print_file_and_line', W_SyntaxError), + lastlineno = readwrite_attrproperty_w('w_lastlineno', W_SyntaxError), ) W_FutureWarning = _new_exception('FutureWarning', W_Warning, From arigo at codespeak.net Wed Dec 8 14:14:33 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 14:14:33 +0100 (CET) Subject: [pypy-svn] r79891 - pypy/trunk/lib-python/modified-2.5.2/test/output Message-ID: <20101208131433.BDA99282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 14:14:32 2010 New Revision: 79891 Modified: pypy/trunk/lib-python/modified-2.5.2/test/output/test_cProfile Log: Fix the expected output. It is now closer to CPython's, but still not completely equal (the disable() method is reported on a differently-named class). Modified: pypy/trunk/lib-python/modified-2.5.2/test/output/test_cProfile ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/output/test_cProfile (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/output/test_cProfile Wed Dec 8 14:14:32 2010 @@ -14,66 +14,66 @@ 4 0.116 0.029 0.120 0.030 test_cProfile.py:78(helper1) 2 0.000 0.000 0.140 0.070 test_cProfile.py:89(helper2_indirect) 8 0.312 0.039 0.400 0.050 test_cProfile.py:93(helper2) - 4 0.000 0.000 0.000 0.000 {append} - 1 0.000 0.000 0.000 0.000 {disable} 12 0.000 0.000 0.012 0.001 {hasattr} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} + 1 0.000 0.000 0.000 0.000 {method 'disable' of 'Profile' objects} 8 0.000 0.000 0.000 0.000 {range} 4 0.000 0.000 0.000 0.000 {sys.exc_info} Ordered by: standard name -Function called... - ncalls tottime cumtime -:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) -test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) - 8 0.000 0.000 {range} -test_cProfile.py:115(__getattr__) -> -test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) - 2 0.040 0.600 test_cProfile.py:60(helper) -test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 20 0.020 0.020 test_cProfile.py:53(mul) -test_cProfile.py:53(mul) -> -test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) - 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) - 6 0.234 0.300 test_cProfile.py:93(helper2) -test_cProfile.py:78(helper1) -> 4 0.000 0.000 {append} - 4 0.000 0.004 {hasattr} - 4 0.000 0.000 {sys.exc_info} -test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) - 2 0.078 0.100 test_cProfile.py:93(helper2) -test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) - 8 0.000 0.008 {hasattr} -{append} -> -{disable} -> -{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) -{range} -> -{sys.exc_info} -> +Function called... + ncalls tottime cumtime +:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) +test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) + 8 0.000 0.000 {range} +test_cProfile.py:115(__getattr__) -> +test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) + 2 0.040 0.600 test_cProfile.py:60(helper) +test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 20 0.020 0.020 test_cProfile.py:53(mul) +test_cProfile.py:53(mul) -> +test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) + 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) + 6 0.234 0.300 test_cProfile.py:93(helper2) +test_cProfile.py:78(helper1) -> 4 0.000 0.004 {hasattr} + 4 0.000 0.000 {method 'append' of 'list' objects} + 4 0.000 0.000 {sys.exc_info} +test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) + 2 0.078 0.100 test_cProfile.py:93(helper2) +test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) + 8 0.000 0.008 {hasattr} +{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) +{method 'append' of 'list' objects} -> +{method 'disable' of 'Profile' objects} -> +{range} -> +{sys.exc_info} -> Ordered by: standard name -Function was called by... - ncalls tottime cumtime -:1() <- -test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) -test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) - 12 0.012 0.012 {hasattr} -test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() -test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) - 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) -test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) -test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) -test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) -test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) -test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) - 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) -{append} <- 4 0.000 0.000 test_cProfile.py:78(helper1) -{disable} <- -{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) - 8 0.000 0.008 test_cProfile.py:93(helper2) -{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) -{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +Function was called by... + ncalls tottime cumtime +:1() <- +test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) +test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) + 12 0.012 0.012 {hasattr} +test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() +test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) + 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) +test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) +test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) +test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) +test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) +test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) + 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) +{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) + 8 0.000 0.008 test_cProfile.py:93(helper2) +{method 'append' of 'list' objects} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +{method 'disable' of 'Profile' objects} <- +{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) +{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) From arigo at codespeak.net Wed Dec 8 15:04:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 15:04:08 +0100 (CET) Subject: [pypy-svn] r79892 - pypy/branch/gc-debug Message-ID: <20101208140408.D4288282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 15:04:06 2010 New Revision: 79892 Added: pypy/branch/gc-debug/ - copied from r79891, pypy/trunk/ Log: A branch in which to improve the GC debugging situation, notably trackgcroot's. From danchr at codespeak.net Wed Dec 8 15:34:36 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Wed, 8 Dec 2010 15:34:36 +0100 (CET) Subject: [pypy-svn] r79893 - pypy/branch/fast-forward/pypy/translator/c/src Message-ID: <20101208143436.02A94282BAD@codespeak.net> Author: danchr Date: Wed Dec 8 15:34:35 2010 New Revision: 79893 Modified: pypy/branch/fast-forward/pypy/translator/c/src/signals.h Log: Include for definition of write(). This fixes the build on Darwin/i386. Modified: pypy/branch/fast-forward/pypy/translator/c/src/signals.h ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/src/signals.h (original) +++ pypy/branch/fast-forward/pypy/translator/c/src/signals.h Wed Dec 8 15:34:35 2010 @@ -11,6 +11,8 @@ #ifdef MS_WINDOWS #include #include +#else +#include #endif #include From arigo at codespeak.net Wed Dec 8 15:38:29 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 15:38:29 +0100 (CET) Subject: [pypy-svn] r79894 - in pypy/branch/gc-debug/pypy/rpython/memory: gc gctransform Message-ID: <20101208143829.D2447282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 15:38:28 2010 New Revision: 79894 Modified: pypy/branch/gc-debug/pypy/rpython/memory/gc/base.py pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py pypy/branch/gc-debug/pypy/rpython/memory/gctransform/framework.py Log: Unless I'm mistaken, making gc.DEBUG settable at run-time is actually very easy and has no performance overhead. Modified: pypy/branch/gc-debug/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-debug/pypy/rpython/memory/gc/base.py Wed Dec 8 15:38:28 2010 @@ -36,6 +36,13 @@ self.finalizer_lock_count = 0 self.run_finalizers = self.AddressDeque() + def post_setup(self): + # More stuff that needs to be initialized when the GC is already + # fully working. (Only called by gctransform/framework for now.) + from pypy.rpython.memory.gc import env + if env.read_from_env('PYPY_GC_DEBUG') > 0: + self.DEBUG = True + def _teardown(self): pass @@ -48,7 +55,8 @@ # The following flag enables costly consistency checks after each # collection. It is automatically set to True by test_gc.py. The # checking logic is translatable, so the flag can be set to True - # here before translation. + # here before translation. At run-time, if PYPY_GC_DEBUG is set, + # then it is also set to True. DEBUG = False def set_query_functions(self, is_varsize, has_gcptr_in_varsize, Modified: pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py Wed Dec 8 15:38:28 2010 @@ -32,6 +32,9 @@ limit. Useful to avoid spending all the time in the GC in very small programs. Defaults to 8 times the nursery. + + PYPY_GC_DEBUG Enable extra checks around collections that are + too slow for normal use. """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -841,7 +844,7 @@ def remember_young_pointer(addr_struct, newvalue): # 'addr_struct' is the address of the object in which we write. # 'newvalue' is the address that we are going to write in there. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") # @@ -878,7 +881,7 @@ # 'addr_array' is the address of the object in which we write, # which must have an array part; 'index' is the index of the # item that is (or contains) the pointer that we write. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_array), "nursery array with GCFLAG_NO_YOUNG_PTRS") objhdr = self.header(addr_array) Modified: pypy/branch/gc-debug/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/gc-debug/pypy/rpython/memory/gctransform/framework.py Wed Dec 8 15:38:28 2010 @@ -189,6 +189,7 @@ # run-time initialization code root_walker.setup_root_walker() gcdata.gc.setup() + gcdata.gc.post_setup() def frameworkgc__teardown(): # run-time teardown code for tests! From arigo at codespeak.net Wed Dec 8 15:43:09 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 15:43:09 +0100 (CET) Subject: [pypy-svn] r79895 - in pypy/trunk/pypy/rpython/memory: gc gctransform Message-ID: <20101208144309.11DB6282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 15:43:08 2010 New Revision: 79895 Modified: pypy/trunk/pypy/rpython/memory/gc/base.py pypy/trunk/pypy/rpython/memory/gc/minimark.py pypy/trunk/pypy/rpython/memory/gctransform/framework.py Log: Merge from branch/gc-debug: at runtime, PYPY_GC_DEBUG=1 enables DEBUG=True in the GC. Modified: pypy/trunk/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/base.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/base.py Wed Dec 8 15:43:08 2010 @@ -36,6 +36,13 @@ self.finalizer_lock_count = 0 self.run_finalizers = self.AddressDeque() + def post_setup(self): + # More stuff that needs to be initialized when the GC is already + # fully working. (Only called by gctransform/framework for now.) + from pypy.rpython.memory.gc import env + if env.read_from_env('PYPY_GC_DEBUG') > 0: + self.DEBUG = True + def _teardown(self): pass @@ -48,7 +55,8 @@ # The following flag enables costly consistency checks after each # collection. It is automatically set to True by test_gc.py. The # checking logic is translatable, so the flag can be set to True - # here before translation. + # here before translation. At run-time, if PYPY_GC_DEBUG is set, + # then it is also set to True. DEBUG = False def set_query_functions(self, is_varsize, has_gcptr_in_varsize, Modified: pypy/trunk/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/minimark.py Wed Dec 8 15:43:08 2010 @@ -32,6 +32,9 @@ limit. Useful to avoid spending all the time in the GC in very small programs. Defaults to 8 times the nursery. + + PYPY_GC_DEBUG Enable extra checks around collections that are + too slow for normal use. """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -841,7 +844,7 @@ def remember_young_pointer(addr_struct, newvalue): # 'addr_struct' is the address of the object in which we write. # 'newvalue' is the address that we are going to write in there. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") # @@ -878,7 +881,7 @@ # 'addr_array' is the address of the object in which we write, # which must have an array part; 'index' is the index of the # item that is (or contains) the pointer that we write. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_array), "nursery array with GCFLAG_NO_YOUNG_PTRS") objhdr = self.header(addr_array) Modified: pypy/trunk/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/framework.py Wed Dec 8 15:43:08 2010 @@ -189,6 +189,7 @@ # run-time initialization code root_walker.setup_root_walker() gcdata.gc.setup() + gcdata.gc.post_setup() def frameworkgc__teardown(): # run-time teardown code for tests! From david at codespeak.net Wed Dec 8 15:55:30 2010 From: david at codespeak.net (david at codespeak.net) Date: Wed, 8 Dec 2010 15:55:30 +0100 (CET) Subject: [pypy-svn] r79896 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . test Message-ID: <20101208145530.5EC82282BAD@codespeak.net> Author: david Date: Wed Dec 8 15:55:28 2010 New Revision: 79896 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Log: Fix malloc_varsize and some other register allocation issues Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Wed Dec 8 15:55:28 2010 @@ -469,7 +469,7 @@ loc = None if isinstance(thing, Const): if isinstance(thing, ConstInt): - box = BoxInt() + box = BoxInt(thing.value) elif isinstance(thing, ConstPtr): box = BoxPtr() else: Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Wed Dec 8 15:55:28 2010 @@ -732,7 +732,8 @@ is_unicode=is_unicode) # compute the length in bytes - length_loc, length_box = self._ensure_value_is_boxed(args[4], regalloc) + forbidden_vars = [srcaddr_box, dstaddr_box] + length_loc, length_box = self._ensure_value_is_boxed(args[4], regalloc, forbidden_vars) args.append(length_box) if is_unicode: forbidden_vars = [srcaddr_box, dstaddr_box] @@ -747,10 +748,10 @@ # call memcpy() self._emit_call(self.memcpy_addr, [dstaddr_box, srcaddr_box, length_box], regalloc) + regalloc.possibly_free_vars(args) regalloc.possibly_free_var(length_box) regalloc.possibly_free_var(dstaddr_box) regalloc.possibly_free_var(srcaddr_box) - regalloc.possibly_free_vars(args) def _load_address(self, sizereg, baseofs, scale, result, baseloc=None): if baseloc is not None: @@ -1031,28 +1032,30 @@ # from: ../x86/regalloc.py:750 # XXX kill this function at some point - def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v, regalloc): - tempbox = TempBox() - size_loc = regalloc.force_allocate_reg(tempbox) - self.mc.gen_load_int(size_loc.value, ofs_items + (v.getint() << scale)) - self._emit_call(self.malloc_func_addr, [tempbox], - regalloc, result=res_v) - loc = regalloc.make_sure_var_in_reg(v, [res_v]) - base_loc = regalloc.loc(res_v) - value_loc = regalloc.loc(v) - regalloc.possibly_free_vars([v, res_v, tempbox]) + def _malloc_varsize(self, ofs_items, ofs_length, itemsize, v, res_v, regalloc): + isize = ConstInt(itemsize) + iofsitems = ConstInt(ofs_items) + boxes = [v, res_v] + vloc, v = self._ensure_value_is_boxed(v, regalloc, [res_v]) + boxes.append(v) + size, size_box = self._ensure_value_is_boxed(isize, regalloc, boxes) + + self._emit_call(self.malloc_func_addr, [size_box], regalloc, result=res_v) + base_loc = regalloc.make_sure_var_in_reg(res_v) + value_loc = regalloc.make_sure_var_in_reg(v) + regalloc.possibly_free_vars(boxes) - # XXX combine with emit_op_setfield_gc operation - size = scale * 2 + self.mc.MUL(size.value, size.value, vloc.value) + if self._check_imm_arg(iofsitems): + self.mc.ADD_ri(size.value, size.value, iofsitems.value) + else: + t, tbox = self._ensure_value_is_boxed(iofsitems, regalloc, boxes) + self.mc.ADD_rr(size.value, size.value, t.value) + regalloc.possibly_free_var(tbox) + + size = itemsize ofs = ofs_length - if size == 4: - self.mc.STR_ri(value_loc.value, base_loc.value, ofs) - elif size == 2: - self.mc.STRH_ri(value_loc.value, base_loc.value, ofs) - elif size == 1: - self.mc.STRB_ri(value_loc.value, base_loc.value, ofs) - else: - assert 0 + self.mc.STR_ri(value_loc.value, base_loc.value, ofs) def emit_op_new(self, op, regalloc, fcond): arglocs = self._prepare_args_for_new_op(op.getdescr(), regalloc) @@ -1098,7 +1101,7 @@ # boehm GC (XXX kill the following code at some point) itemsize, scale, basesize, ofs_length, _ = ( self._unpack_arraydescr(op.getdescr())) - self._malloc_varsize(basesize, ofs_length, scale, + self._malloc_varsize(basesize, ofs_length, itemsize, op.getarg(0), op.result, regalloc) return fcond @@ -1114,7 +1117,7 @@ ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 - self._malloc_varsize(ofs_items, ofs, 1, op.getarg(0), + self._malloc_varsize(ofs_items, ofs, itemsize, op.getarg(0), op.result, regalloc) return fcond @@ -1130,20 +1133,11 @@ # boehm GC (XXX kill the following code at some point) ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) - scale = self._get_unicode_item_scale() - self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0), - op.result, regalloc) - return fcond - - def _get_unicode_item_scale(self): _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) - if itemsize == 4: - return 2 - elif itemsize == 2: - return 1 - else: - raise AssertionError("bad unicode item size") + self._malloc_varsize(ofs_items, ofs, itemsize, op.getarg(0), + op.result, regalloc) + return fcond class ResOpAssembler(GuardOpAssembler, IntOpAsslember, OpAssembler, UnaryIntOpAssembler, Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Wed Dec 8 15:55:28 2010 @@ -5,44 +5,6 @@ from pypy.jit.backend.test.test_random import check_random_function, Random from pypy.jit.metainterp.resoperation import rop -# XXX Remove this list from here once all operations work and use the default -# one -OPERATIONS = test_random.OPERATIONS[:] - -for i in range(4): # make more common - OPERATIONS.append(test_ll_random.GetFieldOperation(rop.GETFIELD_GC)) - OPERATIONS.append(test_ll_random.GetFieldOperation(rop.GETFIELD_GC)) - OPERATIONS.append(test_ll_random.SetFieldOperation(rop.SETFIELD_GC)) - OPERATIONS.append(test_ll_random.NewOperation(rop.NEW)) - OPERATIONS.append(test_ll_random.NewOperation(rop.NEW_WITH_VTABLE)) - - OPERATIONS.append(test_ll_random.GetArrayItemOperation(rop.GETARRAYITEM_GC)) - OPERATIONS.append(test_ll_random.GetArrayItemOperation(rop.GETARRAYITEM_GC)) - OPERATIONS.append(test_ll_random.SetArrayItemOperation(rop.SETARRAYITEM_GC)) - #OPERATIONS.append(test_ll_random.NewArrayOperation(rop.NEW_ARRAY)) - OPERATIONS.append(test_ll_random.ArrayLenOperation(rop.ARRAYLEN_GC)) -# #OPERATIONS.append(test_ll_random.NewStrOperation(rop.NEWSTR)) -# #OPERATIONS.append(test_ll_random.NewUnicodeOperation(rop.NEWUNICODE)) - OPERATIONS.append(test_ll_random.StrGetItemOperation(rop.STRGETITEM)) - OPERATIONS.append(test_ll_random.UnicodeGetItemOperation(rop.UNICODEGETITEM)) - OPERATIONS.append(test_ll_random.StrSetItemOperation(rop.STRSETITEM)) - OPERATIONS.append(test_ll_random.UnicodeSetItemOperation(rop.UNICODESETITEM)) - OPERATIONS.append(test_ll_random.StrLenOperation(rop.STRLEN)) - OPERATIONS.append(test_ll_random.UnicodeLenOperation(rop.UNICODELEN)) -# -for i in range(2): -# OPERATIONS.append(test_ll_random.GuardClassOperation(rop.GUARD_CLASS)) - OPERATIONS.append(test_ll_random.CallOperation(rop.CALL)) - OPERATIONS.append(test_ll_random.RaisingCallOperation(rop.CALL)) - OPERATIONS.append(test_ll_random.RaisingCallOperationGuardNoException(rop.CALL)) - OPERATIONS.append(test_ll_random.RaisingCallOperationWrongGuardException(rop.CALL)) - OPERATIONS.append(test_ll_random.CallOperationException(rop.CALL)) -#OPERATIONS.append(test_ll_random.GuardNonNullClassOperation(rop.GUARD_NONNULL_CLASS)) - - - -LLtypeOperationBuilder.OPERATIONS = OPERATIONS - CPU = getcpuclass() def test_stress(): From arigo at codespeak.net Wed Dec 8 16:13:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 16:13:21 +0100 (CET) Subject: [pypy-svn] r79897 - in pypy/branch/gc-debug/pypy/rpython/lltypesystem: . test Message-ID: <20101208151321.5200E282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 16:13:19 2010 New Revision: 79897 Modified: pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-debug/pypy/rpython/lltypesystem/lltype.py pypy/branch/gc-debug/pypy/rpython/lltypesystem/test/test_llarena.py Log: Add arena_protect(), for now only untranslated. Modified: pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py Wed Dec 8 16:13:19 2010 @@ -26,6 +26,7 @@ self.objectptrs = {} # {offset: ptr-to-container} self.objectsizes = {} # {offset: size} self.freed = False + self.protect_inaccessible = None self.reset(zero) def __repr__(self): @@ -59,6 +60,8 @@ def check(self): if self.freed: raise ArenaError("arena was already freed") + if self.protect_inaccessible is not None: + raise ArenaError("arena is currently arena_protect()ed") def _getid(self): address, length = self.usagemap.buffer_info() @@ -127,6 +130,21 @@ def mark_freed(self): self.freed = True # this method is a hook for tests + def set_protect(self, accessible): + if not accessible: + assert self.protect_inaccessible is None + saved = [] + for ptr in self.objectptrs.values(): + obj = ptr._obj + saved.append((obj, obj._protect())) + self.protect_inaccessible = saved + else: + assert self.protect_inaccessible is not None + saved = self.protect_inaccessible + for obj, storage in saved: + obj._unprotect(storage) + self.protect_inaccessible = None + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -365,6 +383,16 @@ """ return Arena(ptr.arena.nbytes, False).getaddr(0) +def arena_protect(arena_addr, size, accessible): + """For debugging, set or reset memory protection on an arena. + For now, the starting point and size should reference the whole arena. + The value of 'accessible' is a boolean. + """ + arena_addr = getfakearenaaddress(arena_addr) + assert arena_addr.offset == 0 + assert size == arena_addr.arena.nbytes + arena_addr.arena.set_protect(accessible) + # ____________________________________________________________ # # Translation support: the functions above turn into the code below. Modified: pypy/branch/gc-debug/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/gc-debug/pypy/rpython/lltypesystem/lltype.py Wed Dec 8 16:13:19 2010 @@ -1381,6 +1381,15 @@ self._check() # no double-frees self._storage = None + def _protect(self): + result = self._storage + self._free() # no double-frees or double-protects + return result + + def _unprotect(self, saved_storage): + assert self._storage is None + self._storage = saved_storage + def _was_freed(self): if self._storage is None: return True Modified: pypy/branch/gc-debug/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/gc-debug/pypy/rpython/lltypesystem/test/test_llarena.py Wed Dec 8 16:13:19 2010 @@ -6,6 +6,7 @@ from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view from pypy.rpython.lltypesystem.llarena import arena_shrink_obj +from pypy.rpython.lltypesystem.llarena import arena_protect def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -282,3 +283,19 @@ arena_reserve(a, size_gc_header + llmemory.sizeof(S, 10)) arena_shrink_obj(a, size_gc_header + llmemory.sizeof(S, 5)) arena_reset(a, size_gc_header + llmemory.sizeof(S, 5), False) + +def test_arena_protect(): + a = arena_malloc(100, False) + S = lltype.Struct('S', ('x', lltype.Signed)) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 100, accessible=False) + py.test.raises(ArenaError, arena_reserve, a + 48, llmemory.sizeof(S)) + py.test.raises(RuntimeError, "p.x") + py.test.raises(RuntimeError, "p.x = 124") + arena_protect(a, 100, accessible=True) + assert p.x == 123 + p.x = 125 + assert p.x == 125 From david at codespeak.net Wed Dec 8 16:29:36 2010 From: david at codespeak.net (david at codespeak.net) Date: Wed, 8 Dec 2010 16:29:36 +0100 (CET) Subject: [pypy-svn] r79898 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101208152936.45E59282BAD@codespeak.net> Author: david Date: Wed Dec 8 16:29:34 2010 New Revision: 79898 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: Refactor malloc_varsize a bit Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Wed Dec 8 16:29:34 2010 @@ -1033,29 +1033,30 @@ # from: ../x86/regalloc.py:750 # XXX kill this function at some point def _malloc_varsize(self, ofs_items, ofs_length, itemsize, v, res_v, regalloc): - isize = ConstInt(itemsize) - iofsitems = ConstInt(ofs_items) boxes = [v, res_v] + itemsize_box = ConstInt(itemsize) + ofs_items_box = ConstInt(ofs_items) + if self._check_imm_arg(ofs_items_box): + ofs_items_loc = regalloc.convert_to_imm(ofs_items_box) + else: + ofs_items_loc, ofs_items_box = self._ensure_value_is_boxed(ofs_items_box, regalloc, boxes) + boxes.append(ofs_items_box) vloc, v = self._ensure_value_is_boxed(v, regalloc, [res_v]) boxes.append(v) - size, size_box = self._ensure_value_is_boxed(isize, regalloc, boxes) + size, size_box = self._ensure_value_is_boxed(itemsize_box, regalloc, boxes) + self.mc.MUL(size.value, size.value, vloc.value) + if ofs_items_loc.is_imm(): + self.mc.ADD_ri(size.value, size.value, ofs_items_loc.value) + else: + self.mc.ADD_rr(size.value, size.value, ofs_items_loc.value) self._emit_call(self.malloc_func_addr, [size_box], regalloc, result=res_v) + base_loc = regalloc.make_sure_var_in_reg(res_v) value_loc = regalloc.make_sure_var_in_reg(v) - regalloc.possibly_free_vars(boxes) + self.mc.STR_ri(value_loc.value, base_loc.value, ofs_length) - self.mc.MUL(size.value, size.value, vloc.value) - if self._check_imm_arg(iofsitems): - self.mc.ADD_ri(size.value, size.value, iofsitems.value) - else: - t, tbox = self._ensure_value_is_boxed(iofsitems, regalloc, boxes) - self.mc.ADD_rr(size.value, size.value, t.value) - regalloc.possibly_free_var(tbox) - - size = itemsize - ofs = ofs_length - self.mc.STR_ri(value_loc.value, base_loc.value, ofs) + regalloc.possibly_free_vars(boxes) def emit_op_new(self, op, regalloc, fcond): arglocs = self._prepare_args_for_new_op(op.getdescr(), regalloc) From arigo at codespeak.net Wed Dec 8 16:50:11 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 16:50:11 +0100 (CET) Subject: [pypy-svn] r79899 - in pypy/branch/gc-debug/pypy/rpython/lltypesystem: . test Message-ID: <20101208155011.2E118282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 16:50:09 2010 New Revision: 79899 Modified: pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py pypy/branch/gc-debug/pypy/rpython/lltypesystem/test/test_llarena.py Log: Translation of arena_protect(). Modified: pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py Wed Dec 8 16:50:09 2010 @@ -130,8 +130,8 @@ def mark_freed(self): self.freed = True # this method is a hook for tests - def set_protect(self, accessible): - if not accessible: + def set_protect(self, inaccessible): + if inaccessible: assert self.protect_inaccessible is None saved = [] for ptr in self.objectptrs.values(): @@ -383,15 +383,15 @@ """ return Arena(ptr.arena.nbytes, False).getaddr(0) -def arena_protect(arena_addr, size, accessible): +def arena_protect(arena_addr, size, inaccessible): """For debugging, set or reset memory protection on an arena. For now, the starting point and size should reference the whole arena. - The value of 'accessible' is a boolean. + The value of 'inaccessible' is a boolean. """ arena_addr = getfakearenaaddress(arena_addr) assert arena_addr.offset == 0 assert size == arena_addr.arena.nbytes - arena_addr.arena.set_protect(accessible) + arena_addr.arena.set_protect(inaccessible) # ____________________________________________________________ # @@ -503,6 +503,37 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear +if os.name == "posix": + from pypy.translator.tool.cbuild import ExternalCompilationInfo + _eci = ExternalCompilationInfo(includes=['sys/mman.h']) + raw_mprotect = rffi.llexternal('mprotect', + [llmemory.Address, rffi.SIZE_T, rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True, + compilation_info=_eci) + def llimpl_arena_protect(addr, size, inaccessible): + # do some alignment + start = rffi.cast(lltype.Signed, addr) + end = start + size + start = (start + 4095) & ~ 4095 + end = end & ~ 4095 + if end > start: + if inaccessible: + prot = 0 + else: + from pypy.rlib.rmmap import PROT_READ, PROT_WRITE + prot = PROT_READ | PROT_WRITE + raw_mprotect(rffi.cast(llmemory.Address, start), + rffi.cast(rffi.SIZE_T, end - start), + rffi.cast(rffi.INT, prot)) + # ignore potential errors + has_protect = True + +else: + def llimpl_arena_protect(addr, size, inaccessible): + pass + has_protect = False + llimpl_malloc = rffi.llexternal('malloc', [lltype.Signed], llmemory.Address, sandboxsafe=True, _nowrapper=True) @@ -572,6 +603,11 @@ 'll_arena.arena_new_view', llimpl=llimpl_arena_new_view, llfakeimpl=arena_new_view, sandboxsafe=True) +register_external(arena_protect, [llmemory.Address, lltype.Signed, + lltype.Bool], lltype.Void, + 'll_arena.arena_protect', llimpl=llimpl_arena_protect, + llfakeimpl=arena_protect, sandboxsafe=True) + def llimpl_getfakearenaaddress(addr): return addr register_external(getfakearenaaddress, [llmemory.Address], llmemory.Address, Modified: pypy/branch/gc-debug/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/gc-debug/pypy/rpython/lltypesystem/test/test_llarena.py Wed Dec 8 16:50:09 2010 @@ -6,7 +6,8 @@ from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view from pypy.rpython.lltypesystem.llarena import arena_shrink_obj -from pypy.rpython.lltypesystem.llarena import arena_protect +from pypy.rpython.lltypesystem.llarena import arena_protect, has_protect +from pypy.translator.c.test import test_genc, test_standalone def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -266,8 +267,7 @@ assert res == 42 def test_compiled(): - from pypy.translator.c.test.test_genc import compile - fn = compile(test_look_inside_object, []) + fn = test_genc.compile(test_look_inside_object, []) res = fn() assert res == 42 @@ -291,11 +291,43 @@ p = llmemory.cast_adr_to_ptr(a, lltype.Ptr(S)) p.x = 123 assert p.x == 123 - arena_protect(a, 100, accessible=False) + arena_protect(a, 100, True) py.test.raises(ArenaError, arena_reserve, a + 48, llmemory.sizeof(S)) py.test.raises(RuntimeError, "p.x") py.test.raises(RuntimeError, "p.x = 124") - arena_protect(a, 100, accessible=True) + arena_protect(a, 100, False) assert p.x == 123 p.x = 125 assert p.x == 125 + + +class TestStandalone(test_standalone.StandaloneTests): + def test_compiled_arena_protect(self): + import os + from pypy.translator.c.test.test_genc import compile + S = lltype.Struct('S', ('x', lltype.Signed)) + # + def fn(argv): + testrun = int(argv[1]) + a = arena_malloc(65536, False) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a + 23432, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 65536, True) + result = 0 + if testrun == 1: + print p.x # segfault + if testrun == 2: + p.x = 124 # segfault + arena_protect(a, 65536, False) + p.x += 10 + print p.x + return 0 + # + t, cbuilder = self.compile(fn) + data = cbuilder.cmdexec('0') + assert data == '133\n' + if has_protect: + cbuilder.cmdexec('1', expect_crash=True) + cbuilder.cmdexec('2', expect_crash=True) From arigo at codespeak.net Wed Dec 8 17:14:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 17:14:02 +0100 (CET) Subject: [pypy-svn] r79900 - pypy/branch/gc-debug/pypy/rpython/memory/gc Message-ID: <20101208161402.C35B9282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 17:14:01 2010 New Revision: 79900 Modified: pypy/branch/gc-debug/pypy/rpython/memory/gc/base.py pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py Log: - Use PYPY_GC_DEBUG=2 to also do debug checks after minor collections (slow). - Implemented a scheme where, if PYPY_GC_DEBUG is set, we allocate 23 nurseries instead of just 1. After every minor collection we switch to the next one. The unused nurseries are mprotect()ed. This should allow us to quickly spot places that still use pointers to one of the old nurseries. Modified: pypy/branch/gc-debug/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/gc-debug/pypy/rpython/memory/gc/base.py Wed Dec 8 17:14:01 2010 @@ -40,8 +40,7 @@ # More stuff that needs to be initialized when the GC is already # fully working. (Only called by gctransform/framework for now.) from pypy.rpython.memory.gc import env - if env.read_from_env('PYPY_GC_DEBUG') > 0: - self.DEBUG = True + self.DEBUG = env.read_from_env('PYPY_GC_DEBUG') def _teardown(self): pass Modified: pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py Wed Dec 8 17:14:01 2010 @@ -3,7 +3,9 @@ Environment variables can be used to fine-tune the following parameters: PYPY_GC_NURSERY The nursery size. Defaults to half the size of - the L2 cache. Try values like '1.2MB'. + the L2 cache. Try values like '1.2MB'. Minimum + is about 64KB. Use 1 to force every malloc to + do a minor collection for debugging. PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', which means trigger a major collection when the @@ -34,7 +36,9 @@ times the nursery. PYPY_GC_DEBUG Enable extra checks around collections that are - too slow for normal use. + too slow for normal use. Values are 0 (off), + 1 (on major collections) or 2 (also on minor + collections). """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -227,6 +231,7 @@ self.nursery_free = NULL self.nursery_top = NULL self.debug_always_do_minor_collect = False + self.debug_rotating_nurseries = None # # The ArenaCollection() handles the nonmovable objects allocation. if ArenaCollectionClass is None: @@ -335,17 +340,24 @@ self.allocate_nursery() - def allocate_nursery(self): - debug_start("gc-set-nursery-size") - debug_print("nursery size:", self.nursery_size) + def _nursery_memory_size(self): + extra = self.nonlarge_gcptrs_max + 1 + return self.nursery_size + extra + + def _alloc_nursery(self): # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize_clear(). The few extra pages are never used # anyway so it doesn't even count. - extra = self.nonlarge_gcptrs_max + 1 - self.nursery = llarena.arena_malloc(self.nursery_size + extra, 2) - if not self.nursery: + nursery = llarena.arena_malloc(self._nursery_memory_size(), 2) + if not nursery: raise MemoryError("cannot allocate nursery") + return nursery + + def allocate_nursery(self): + debug_start("gc-set-nursery-size") + debug_print("nursery size:", self.nursery_size) + self.nursery = self._alloc_nursery() # the current position in the nursery: self.nursery_free = self.nursery # the end of the nursery: @@ -379,6 +391,39 @@ return bounded + def post_setup(self): + # set up extra stuff for PYPY_GC_DEBUG. + MovingGCBase.post_setup(self) + if self.DEBUG and llarena.has_protect: + # gc debug mode: allocate 23 nurseries instead of just 1, + # and use them alternatively, while mprotect()ing the unused + # ones to detect invalid access. + debug_start("gc-debug") + self.debug_rotating_nurseries = [] + for i in range(22): + nurs = self._alloc_nursery() + llarena.arena_protect(nurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(nurs) + debug_print("allocated", len(self.debug_rotating_nurseries), + "extra nurseries") + debug_stop("gc-debug") + + def debug_rotate_nursery(self): + if self.debug_rotating_nurseries is not None: + debug_start("gc-debug") + oldnurs = self.nursery + llarena.arena_protect(oldnurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(oldnurs) + # + newnurs = self.debug_rotating_nurseries.pop(0) + llarena.arena_protect(newnurs, self._nursery_memory_size(), False) + self.nursery = newnurs + self.nursery_top = self.nursery + self.nursery_size + debug_print("switching to nursery", self.nursery, + "size", self.nursery_size) + debug_stop("gc-debug") + + def malloc_fixedsize_clear(self, typeid, size, can_collect=True, needs_finalizer=False, contains_weakptr=False): ll_assert(can_collect, "!can_collect") @@ -1003,13 +1048,14 @@ # All live nursery objects are out, and the rest dies. Fill # the whole nursery with zero and reset the current nursery pointer. llarena.arena_reset(self.nursery, self.nursery_size, 2) + self.debug_rotate_nursery() self.nursery_free = self.nursery # debug_print("minor collect, total memory used:", self.get_total_memory_used()) + if self.DEBUG >= 2: + self.debug_check_consistency() # expensive! debug_stop("gc-minor") - if 0: # not we_are_translated(): - self.debug_check_consistency() # xxx expensive! def collect_roots_in_nursery(self): From agaynor at codespeak.net Wed Dec 8 17:18:35 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 8 Dec 2010 17:18:35 +0100 (CET) Subject: [pypy-svn] r79901 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20101208161835.D1A6C282BAD@codespeak.net> Author: agaynor Date: Wed Dec 8 17:18:34 2010 New Revision: 79901 Modified: pypy/branch/fast-forward/pypy/objspace/std/ropeobject.py Log: Fixed ropeobject tests. Modified: pypy/branch/fast-forward/pypy/objspace/std/ropeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/ropeobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/ropeobject.py Wed Dec 8 17:18:34 2010 @@ -15,7 +15,7 @@ from pypy.rlib import rope from pypy.objspace.std.stringobject import mod__String_ANY as mod__Rope_ANY,\ - _upper, _lower + _upper, _lower, DEFAULT_NOOP_TABLE class W_RopeObject(W_Object): from pypy.objspace.std.stringtype import str_typedef as typedef @@ -830,11 +830,14 @@ which must be a string of length 256""" # XXX CPython accepts buffers, too, not sure what we should do - table = space.str_w(w_table) - if len(table) != 256: - raise OperationError( - space.w_ValueError, - space.wrap("translation table must be 256 characters long")) + if space.is_w(w_table, space.w_None): + table = DEFAULT_NOOP_TABLE + else: + table = space.str_w(w_table) + if len(table) != 256: + raise OperationError( + space.w_ValueError, + space.wrap("translation table must be 256 characters long")) node = w_string._node chars = [] From arigo at codespeak.net Wed Dec 8 18:07:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Wed, 8 Dec 2010 18:07:21 +0100 (CET) Subject: [pypy-svn] r79902 - in pypy/branch/gc-debug/pypy: rlib rpython/lltypesystem Message-ID: <20101208170721.72FBC282BAD@codespeak.net> Author: arigo Date: Wed Dec 8 18:07:19 2010 New Revision: 79902 Modified: pypy/branch/gc-debug/pypy/rlib/rmmap.py pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py Log: Implement arena_protect() on Windows. Untested. Modified: pypy/branch/gc-debug/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/gc-debug/pypy/rlib/rmmap.py (original) +++ pypy/branch/gc-debug/pypy/rlib/rmmap.py Wed Dec 8 18:07:19 2010 @@ -67,7 +67,7 @@ constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE', - 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE'] + 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS'] for name in constant_names: setattr(CConfig, name, rffi_platform.ConstantInteger(name)) Modified: pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/gc-debug/pypy/rpython/lltypesystem/llarena.py Wed Dec 8 18:07:19 2010 @@ -511,27 +511,34 @@ rffi.INT, sandboxsafe=True, _nowrapper=True, compilation_info=_eci) - def llimpl_arena_protect(addr, size, inaccessible): - # do some alignment - start = rffi.cast(lltype.Signed, addr) - end = start + size - start = (start + 4095) & ~ 4095 - end = end & ~ 4095 - if end > start: - if inaccessible: - prot = 0 - else: - from pypy.rlib.rmmap import PROT_READ, PROT_WRITE - prot = PROT_READ | PROT_WRITE - raw_mprotect(rffi.cast(llmemory.Address, start), - rffi.cast(rffi.SIZE_T, end - start), - rffi.cast(rffi.INT, prot)) - # ignore potential errors + def llimpl_protect(addr, size, inaccessible): + if inaccessible: + prot = 0 + else: + from pypy.rlib.rmmap import PROT_READ, PROT_WRITE + prot = PROT_READ | PROT_WRITE + raw_mprotect(addr, rffi.cast(rffi.SIZE_T, size), + rffi.cast(rffi.INT, prot)) + # ignore potential errors + has_protect = True + +elif os.name == 'nt': + def llimpl_protect(addr, size, inaccessible): + from pypy.rlib.rmmap import VirtualProtect, LPDWORD + if inaccessible: + from pypy.rlib.rmmap import PAGE_NOACCESS as newprotect + else: + from pypy.rlib.rmmap import PAGE_READWRITE as newprotect + arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') + VirtualProtect(rffi.cast(rffi.VOIDP, addr), + rffi.cast(rffi.SIZE_T, size), + newprotect, + arg) + # ignore potential errors + lltype.free(arg, flavor='raw') has_protect = True else: - def llimpl_arena_protect(addr, size, inaccessible): - pass has_protect = False @@ -603,6 +610,16 @@ 'll_arena.arena_new_view', llimpl=llimpl_arena_new_view, llfakeimpl=arena_new_view, sandboxsafe=True) +def llimpl_arena_protect(addr, size, inaccessible): + if has_protect: + # do some alignment + start = rffi.cast(lltype.Signed, addr) + end = start + size + start = (start + 4095) & ~ 4095 + end = end & ~ 4095 + if end > start: + llimpl_protect(rffi.cast(llmemory.Address, start), end-start, + inaccessible) register_external(arena_protect, [llmemory.Address, lltype.Signed, lltype.Bool], lltype.Void, 'll_arena.arena_protect', llimpl=llimpl_arena_protect, From dan at codespeak.net Wed Dec 8 18:22:36 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 8 Dec 2010 18:22:36 +0100 (CET) Subject: [pypy-svn] r79903 - in pypy/branch/psycopg2compatibility/pypy/module/cpyext: . test Message-ID: <20101208172236.0DA1D282BAD@codespeak.net> Author: dan Date: Wed Dec 8 18:22:34 2010 New Revision: 79903 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py pypy/branch/psycopg2compatibility/pypy/module/cpyext/stubs.py pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Log: Added support for PySequence_SetItem. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py Wed Dec 8 18:22:34 2010 @@ -6,6 +6,9 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.objspace.std import listobject, tupleobject +from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem +from pypy.module.cpyext.object import Py_IncRef + @cpython_api([PyObject, Py_ssize_t], PyObject) def PySequence_Repeat(space, w_obj, count): """Return the result of repeating sequence object o count times, or NULL on @@ -124,3 +127,24 @@ """ return space.iter(w_seq) + at cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) +def PySequence_SetItem(space, w_o, i, w_v): + """Assign object v to the ith element of o. Returns -1 on failure. This + is the equivalent of the Python statement o[i] = v. This function does + not steal a reference to v. + + This function used an int type for i. This might require + changes in your code for properly supporting 64-bit systems.""" + + Py_IncRef(space, w_v) # XXX: seriously CPython, why should one Py*_SetItem steal but not another!? + if PyTuple_Check(space, w_o): + return PyTuple_SetItem(space, w_o, i, w_v) + + try: + space.setitem(w_o, space.wrap(i), w_v) + return 0 + except OperationError, e: + if e.match(space, space.w_IndexError): + return -1 + else: + raise Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/stubs.py Wed Dec 8 18:22:34 2010 @@ -2223,16 +2223,6 @@ changes in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PySequence_SetItem(space, o, i, v): - """Assign object v to the ith element of o. Returns -1 on failure. This - is the equivalent of the Python statement o[i] = v. This function does - not steal a reference to v. - - This function used an int type for i. This might require - changes in your code for properly supporting 64-bit systems.""" - raise NotImplementedError - @cpython_api([PyObject, Py_ssize_t], rffi.INT_real, error=-1) def PySequence_DelItem(space, o, i): """Delete the ith element of object o. Returns -1 on failure. This is the Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Wed Dec 8 18:22:34 2010 @@ -70,3 +70,21 @@ assert space.unwrap(space.next(w_iter)) == 2 exc = raises(OperationError, space.next, w_iter) assert exc.value.match(space, space.w_StopIteration) + + def test_setitem(self, space, api): + value = api.PyInt_FromLong(42) + tup = api.PyTuple_New(1) + + result = api.PySequence_SetItem(tup, 0, value) + assert result != -1 + + assert space.eq_w(space.getitem(tup, space.wrap(0)), value) + + l = api.PyList_New(1) + + result = api.PySequence_SetItem(l, 0, value) + assert result != -1 + + assert space.eq_w(space.getitem(l, space.wrap(0)), value) + api.Py_DecRef(value) + From dan at codespeak.net Wed Dec 8 18:24:02 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 8 Dec 2010 18:24:02 +0100 (CET) Subject: [pypy-svn] r79904 - pypy/branch/psycopg2compatibility/pypy/module/cpyext Message-ID: <20101208172402.CA64B282BAD@codespeak.net> Author: dan Date: Wed Dec 8 18:24:00 2010 New Revision: 79904 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py Log: use debug_print instead of dumping to stdout Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/pyobject.py Wed Dec 8 18:24:00 2010 @@ -11,6 +11,8 @@ from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.rpython.annlowlevel import llhelper +from pypy.rlib.debug import debug_start, debug_print, debug_stop + #________________________________________________________ # type description @@ -235,7 +237,9 @@ ref = self.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) if not ref: if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone:", w_obj + debug_start('cpyext-refcount') + debug_print("Borrowed object is already gone:" + str(w_obj)) + debug_stop('cpyext-refcount') return containee_ptr = rffi.cast(ADDR, ref) @@ -256,12 +260,11 @@ frame_stackdepth = kwargs.pop("frame_stackdepth", 2) assert not kwargs frame = sys._getframe(frame_stackdepth) - sys.stderr.write(cpy_logger.get_indent_string()) - sys.stderr.write(frame.f_code.co_name) - sys.stderr.write(' ') - for arg in args: - print >>sys.stderr, arg, - print >>sys.stderr + debug_start('cpyext-refcount') + debug_print(cpy_logger.get_indent_string()) + debug_print(frame.f_code.co_name) + debug_print(' '.join([frame.f_code.co_name] + [str(arg) for arg in args])) + debug_stop('cpyext-refcount') def create_ref(space, w_obj, itemcount=0): """ @@ -377,7 +380,9 @@ if not we_are_translated() and obj.c_ob_refcnt < 0: message = "Negative refcount for obj %s with type %s" % ( obj, rffi.charp2str(obj.c_ob_type.c_tp_name)) - print >>sys.stderr, message + debug_start('cpyext-refcount') + debug_print(message) + debug_stop('cpyext-refcount') assert False, message @cpython_api([PyObject], lltype.Void) From dan at codespeak.net Wed Dec 8 18:28:19 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 8 Dec 2010 18:28:19 +0100 (CET) Subject: [pypy-svn] r79905 - pypy/branch/psycopg2compatibility/pypy/module/cpyext Message-ID: <20101208172819.6DC2F282BAD@codespeak.net> Author: dan Date: Wed Dec 8 18:28:17 2010 New Revision: 79905 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/api.py Log: Changed way too much for one commit... Always decrement cpyext logging indentation, even in case of exception. Can disable logging at runtime, such as when the logger calls PyObject_Repr. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/api.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/api.py Wed Dec 8 18:28:17 2010 @@ -41,14 +41,18 @@ from pypy.tool.call_logger import CallLogger, SkipArgument from pypy.rlib.objectmodel import specialize +from pypy.rlib.debug import debug_start, debug_stop + def debug_func(category): def decorator(f): from pypy.rlib.debug import debug_start, debug_stop @wraps(f) def debugged(*args): - debug_start(category) - result = f(*args) - debug_stop(category) + try: + debug_start(category) + result = f(*args) + finally: + debug_stop(category) return result return debugged return decorator @@ -56,6 +60,7 @@ class CPyExtCallLogger(object): def __init__(self): self.indentation = 0 + self.enabled = True def get_indent_string(self, depth=None): if depth is None: @@ -63,46 +68,54 @@ return ' ' * depth - def log(self, logstr, depth=0): + def log(self, logstr, depth=None): + if depth is None: + depth = self.indentation + from pypy.rlib.debug import debug_print - debug_print(self.get_indent_string() + logstr) + debug_print(self.get_indent_string(depth) + logstr) def log_cpyext_rpython_call(self, f): - if DEBUG_WRAPPER: - name = f.__name__ - #@debug_func('cpyext') - @wraps(f) - def wrapped(space, *args): + name = f.__name__ + @wraps(f) + def wrapped(space, *args): + if self.enabled: argstrs = [repr(x) for x in args] argstr = ', '.join(argstrs) - self.log("%s(%s)" % (name, argstr), depth=self.indentation) - self.indentation += 1 - result = f(space, *args) - self.indentation -= 1 - self.log("%s(%s)->%s" % (name, argstr, repr(result)), depth=self.indentation) + debug_start('cpyext') + self.log("%s(%s)" % (name, argstr)) + try: + self.indentation += 1 + result = f(space, *args) + finally: + self.indentation -= 1 + debug_stop('cpyext') + self.log("%s(%s)->%s" % (name, argstr, repr(result))) return result - return wrapped - else: - return f + else: + return f(space, *args) + return wrapped def log_cpyext_call(self, space, f): - if DEBUG_WRAPPER: - from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.object import PyObject_Repr - from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef - api_function = f.api_func - argtypes = api_function.argtypes - result_type = api_function.restype - argwrapped = [name.startswith("w_") for name in api_function.argnames] - assert len(api_function.argtypes) == len(api_function.argnames) - argtypes_enum_ui = unrolling_iterable(enumerate(zip(api_function.argtypes, - [name.startswith("w_") for name in api_function.argnames]), start=1)) - name = f.__name__ # XXX: above line was overwriting name "name" - - #@debug_func('cpyext') - @wraps(f) - def wrapped(*args): + from pypy.module.cpyext.object import PyObject_Repr + from pypy.module.cpyext.pyobject import Py_IncRef, Py_DecRef + + api_function = f.api_func + argtypes = api_function.argtypes + + assert len(api_function.argtypes) == len(api_function.argnames) + argtypes_enum_ui = unrolling_iterable(enumerate(zip(argtypes, + [arg_name.startswith("w_") for arg_name in api_function.argnames]), start=1)) + + result_type = api_function.restype + name = f.__name__ + + @wraps(f) + def wrapped(*args): + if self.enabled: + # XXX: looks suspiciously like wrapper + self.enabled = False argstrs = [] for i, (argtype, is_wrapped) in argtypes_enum_ui: arg = args[i] @@ -113,24 +126,28 @@ py_repr = PyObject_Repr(space, w_arg) Py_DecRef(space, w_arg) argstrs.append(space.str_w(py_repr)) - else: - argstrs.append(repr(w_arg)) - else: - argstrs.append(repr(arg)) + continue + + argstrs.append(repr(arg)) argstr = ', '.join(argstrs) + self.enabled = True - self.log("%s(%s)" % (name, argstr), depth=self.indentation) - self.indentation += 1 - result = f(*args) - self.indentation -= 1 + debug_start('cpyext') + self.log("%s(%s)" % (name, argstr)) + try: + self.indentation += 1 + result = f(*args) + finally: + self.indentation -= 1 + debug_stop('cpyext') result_format = repr(result) # TODO: format nicer! - self.log("%s(%s)->%s" % (name, argstr, result_format), depth=self.indentation) + self.log("%s(%s)->%s" % (name, argstr, result_format)) return result + else: + return f(*args) - return wrapped - else: - return f + return wrapped cpy_logger = CPyExtCallLogger() log_call = cpy_logger.log_cpyext_call @@ -628,8 +645,12 @@ if failed: error_value = callable.api_func.error_value if error_value is CANNOT_FAIL: - raise SystemError("The function '%s' was not supposed to fail" - % (callable.__name__,)) + if not we_are_translated(): + raise SystemError("The function '%s' was not supposed to fail. Failed with %s" + % (callable.__name__, e)) + else: + raise SystemError("The function '%s' was not supposed to fail" + % (callable.__name__,)) retval = error_value elif is_PyObject(callable.api_func.restype): From agaynor at codespeak.net Wed Dec 8 18:39:44 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 8 Dec 2010 18:39:44 +0100 (CET) Subject: [pypy-svn] r79906 - pypy/branch/fast-forward/pypy/doc/config Message-ID: <20101208173944.4AD5E282BAD@codespeak.net> Author: agaynor Date: Wed Dec 8 18:39:42 2010 New Revision: 79906 Added: pypy/branch/fast-forward/pypy/doc/config/objspace.usemodules._warnings.txt Log: Added a doc file for the _warning usemodule config. Added: pypy/branch/fast-forward/pypy/doc/config/objspace.usemodules._warnings.txt ============================================================================== --- (empty file) +++ pypy/branch/fast-forward/pypy/doc/config/objspace.usemodules._warnings.txt Wed Dec 8 18:39:42 2010 @@ -0,0 +1 @@ +Use the '_warning' module. This module is expected to be working and is included by default. From agaynor at codespeak.net Wed Dec 8 18:58:03 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Wed, 8 Dec 2010 18:58:03 +0100 (CET) Subject: [pypy-svn] r79907 - pypy/trunk/pypy/interpreter/test Message-ID: <20101208175803.B2262282BAD@codespeak.net> Author: agaynor Date: Wed Dec 8 18:57:55 2010 New Revision: 79907 Modified: pypy/trunk/pypy/interpreter/test/test_function.py Log: A failing test for call(*0) or call(**0) Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Wed Dec 8 18:57:55 2010 @@ -6,7 +6,7 @@ from pypy.interpreter.argument import Arguments -class AppTestFunctionIntrospection: +class AppTestFunctionIntrospection: def test_attributes(self): globals()['__name__'] = 'mymodulename' def f(): pass @@ -88,10 +88,10 @@ def f(*args): return 42 raises(TypeError, "dir.func_code = f.func_code") - raises(TypeError, "list.append.im_func.func_code = f.func_code") + raises(TypeError, "list.append.im_func.func_code = f.func_code") -class AppTestFunction: +class AppTestFunction: def test_simple_call(self): def func(arg1, arg2): return arg1, arg2 @@ -116,7 +116,7 @@ assert res[2] == 333 raises(TypeError, func) - raises(TypeError, func, 1, 2, 3, 4) + raises(TypeError, func, 1, 2, 3, 4) def test_simple_varargs(self): def func(arg1, *args): @@ -127,7 +127,7 @@ res = func(23, *(42,)) assert res[0] == 23 - assert res[1] == (42,) + assert res[1] == (42,) def test_simple_kwargs(self): def func(arg1, **kwargs): @@ -205,7 +205,7 @@ func(**{'self': 23}) assert False except TypeError: - pass + pass def test_kwargs_confusing_name(self): def func(self): # 'self' conflicts with the interp-level @@ -287,7 +287,7 @@ # on function types raises(ValueError, type(f).__setstate__, f, (1, 2, 3)) -class AppTestMethod: +class AppTestMethod: def test_simple_call(self): class A(object): def func(self, arg2): @@ -308,7 +308,7 @@ res = a.func(*(42,)) assert res[0] is a - assert res[1] == (42,) + assert res[1] == (42,) def test_obscure_varargs(self): class A(object): @@ -321,14 +321,14 @@ res = a.func(*(42,)) assert res[0] is a - assert res[1] == 42 + assert res[1] == 42 def test_simple_kwargs(self): class A(object): def func(self, **kwargs): return self, kwargs a = A() - + res = a.func(value=42) assert res[0] is a assert res[1] == {'value': 42} @@ -382,19 +382,19 @@ assert hash(C.m) == hash(D.m) assert hash(c.m) == hash(c.m) - def test_method_repr(self): - class A(object): - def f(self): + def test_method_repr(self): + class A(object): + def f(self): pass assert repr(A.f) == "" - assert repr(A().f).startswith(">") + assert repr(A().f).startswith(">") class B: def f(self): pass assert repr(B.f) == "" assert repr(B().f).startswith(">") + assert repr(A().f).endswith(">>") def test_method_call(self): @@ -487,14 +487,21 @@ def f(): pass raises(TypeError, new.instancemethod, f, None) + def test_empty_arg_kwarg_call(self): + def f(): + pass -class TestMethod: + raises(TypeError, lambda: f(*0)) + raises(TypeError, lambda: f(**0)) + + +class TestMethod: def setup_method(self, method): def c(self, bar): return bar code = PyCode._from_code(self.space, c.func_code) self.fn = Function(self.space, code, self.space.newdict()) - + def test_get(self): space = self.space w_meth = descr_function_get(space, self.fn, space.wrap(5), space.type(space.wrap(5))) @@ -552,7 +559,7 @@ def test_call_function(self): space = self.space - + d = {} for i in range(10): args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")" @@ -574,14 +581,14 @@ code.funcrun = bomb code.funcrun_obj = bomb - args_w = map(space.wrap, range(i)) + args_w = map(space.wrap, range(i)) w_res = space.call_function(fn, *args_w) check = space.is_true(space.eq(w_res, space.wrap(res))) assert check def test_flatcall(self): space = self.space - + def f(a): return a code = PyCode._from_code(self.space, f.func_code) @@ -608,7 +615,7 @@ def test_flatcall_method(self): space = self.space - + def f(self, a): return a code = PyCode._from_code(self.space, f.func_code) @@ -636,7 +643,7 @@ def test_flatcall_default_arg(self): space = self.space - + def f(a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -665,7 +672,7 @@ def test_flatcall_default_arg_method(self): space = self.space - + def f(self, a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -688,7 +695,7 @@ y = A().m(x) b = A().m z = b(x) - return y+10*z + return y+10*z """) assert space.eq_w(w_res, space.wrap(44)) From dan at codespeak.net Wed Dec 8 19:25:15 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 8 Dec 2010 19:25:15 +0100 (CET) Subject: [pypy-svn] r79908 - in pypy/branch/psycopg2compatibility/pypy/module/cpyext: . test Message-ID: <20101208182515.36FE7282BAD@codespeak.net> Author: dan Date: Wed Dec 8 19:25:12 2010 New Revision: 79908 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Log: Don't allow PySequence_SetItem on tuples, change the the test to reflect this. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py Wed Dec 8 19:25:12 2010 @@ -7,7 +7,7 @@ from pypy.objspace.std import listobject, tupleobject from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem -from pypy.module.cpyext.object import Py_IncRef +from pypy.module.cpyext.object import Py_IncRef, Py_DecRef @cpython_api([PyObject, Py_ssize_t], PyObject) def PySequence_Repeat(space, w_obj, count): @@ -136,15 +136,10 @@ This function used an int type for i. This might require changes in your code for properly supporting 64-bit systems.""" - Py_IncRef(space, w_v) # XXX: seriously CPython, why should one Py*_SetItem steal but not another!? - if PyTuple_Check(space, w_o): - return PyTuple_SetItem(space, w_o, i, w_v) - try: + Py_IncRef(space, w_v) space.setitem(w_o, space.wrap(i), w_v) return 0 - except OperationError, e: - if e.match(space, space.w_IndexError): - return -1 - else: - raise + except: + Py_DecRef(space, w_v) + raise Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Wed Dec 8 19:25:12 2010 @@ -75,10 +75,8 @@ value = api.PyInt_FromLong(42) tup = api.PyTuple_New(1) - result = api.PySequence_SetItem(tup, 0, value) - assert result != -1 - - assert space.eq_w(space.getitem(tup, space.wrap(0)), value) + exc = raises(OperationError, sequence.PySequence_SetItem, space, tup, 0, value) + assert exc.value.match(space, space.w_TypeError) l = api.PyList_New(1) @@ -87,4 +85,3 @@ assert space.eq_w(space.getitem(l, space.wrap(0)), value) api.Py_DecRef(value) - From dan at codespeak.net Wed Dec 8 19:40:53 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 8 Dec 2010 19:40:53 +0100 (CET) Subject: [pypy-svn] r79909 - in pypy/branch/psycopg2compatibility/pypy/module/cpyext: . test Message-ID: <20101208184053.BE292282BAD@codespeak.net> Author: dan Date: Wed Dec 8 19:40:51 2010 New Revision: 79909 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Log: Using PyErr_\* to test for exceptions in the test, don't incref value. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py Wed Dec 8 19:40:51 2010 @@ -136,10 +136,5 @@ This function used an int type for i. This might require changes in your code for properly supporting 64-bit systems.""" - try: - Py_IncRef(space, w_v) - space.setitem(w_o, space.wrap(i), w_v) - return 0 - except: - Py_DecRef(space, w_v) - raise + space.setitem(w_o, space.wrap(i), w_v) + return 0 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Wed Dec 8 19:40:51 2010 @@ -75,8 +75,10 @@ value = api.PyInt_FromLong(42) tup = api.PyTuple_New(1) - exc = raises(OperationError, sequence.PySequence_SetItem, space, tup, 0, value) - assert exc.value.match(space, space.w_TypeError) + result = api.PySequence_SetItem(tup, 0, value) + assert result == -1 + assert api.PyErr_Occurred() + assert api.PyErr_ExceptionMatches(space.w_TypeError) l = api.PyList_New(1) @@ -84,4 +86,9 @@ assert result != -1 assert space.eq_w(space.getitem(l, space.wrap(0)), value) - api.Py_DecRef(value) + + result = api.PySequence_SetItem(l, 3, value) + assert result == -1 + assert api.PyErr_Occurred() + assert api.PyErr_ExceptionMatches(space.w_IndexError) + api.PyErr_Clear() From dan at codespeak.net Wed Dec 8 20:14:55 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 8 Dec 2010 20:14:55 +0100 (CET) Subject: [pypy-svn] r79910 - in pypy/branch/psycopg2compatibility/pypy/module/cpyext: . test Message-ID: <20101208191455.DC318282BE0@codespeak.net> Author: dan Date: Wed Dec 8 20:14:53 2010 New Revision: 79910 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Log: Now reject dictionaries, test for this. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py Wed Dec 8 20:14:53 2010 @@ -9,6 +9,8 @@ from pypy.module.cpyext.tupleobject import PyTuple_Check, PyTuple_SetItem from pypy.module.cpyext.object import Py_IncRef, Py_DecRef +from pypy.module.cpyext.dictobject import PyDict_Check + @cpython_api([PyObject, Py_ssize_t], PyObject) def PySequence_Repeat(space, w_obj, count): """Return the result of repeating sequence object o count times, or NULL on @@ -136,5 +138,7 @@ This function used an int type for i. This might require changes in your code for properly supporting 64-bit systems.""" + if PyDict_Check(space, w_o) or not PySequence_Check(space, w_o): + raise OperationError(space.w_TypeError, "object doesn't support sequence assignment") # FIXME: format like CPython space.setitem(w_o, space.wrap(i), w_v) return 0 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Wed Dec 8 20:14:53 2010 @@ -72,23 +72,22 @@ assert exc.value.match(space, space.w_StopIteration) def test_setitem(self, space, api): - value = api.PyInt_FromLong(42) - tup = api.PyTuple_New(1) - - result = api.PySequence_SetItem(tup, 0, value) - assert result == -1 - assert api.PyErr_Occurred() - assert api.PyErr_ExceptionMatches(space.w_TypeError) + def expect_error(w_o, i=0, v=42, w_err=space.w_TypeError): + value = space.wrap(v) + result = api.PySequence_SetItem(w_o, i, value) + assert result == -1 + assert api.PyErr_Occurred() + assert api.PyErr_ExceptionMatches(w_err) + api.PyErr_Clear() l = api.PyList_New(1) - - result = api.PySequence_SetItem(l, 0, value) + w_value = space.wrap(42) + result = api.PySequence_SetItem(l, 0, w_value) assert result != -1 + assert space.eq_w(space.getitem(l, space.wrap(0)), w_value) - assert space.eq_w(space.getitem(l, space.wrap(0)), value) + expect_error(l, 3, + w_err=space.w_IndexError) - result = api.PySequence_SetItem(l, 3, value) - assert result == -1 - assert api.PyErr_Occurred() - assert api.PyErr_ExceptionMatches(space.w_IndexError) - api.PyErr_Clear() + expect_error(api.PyTuple_New(1)) # TypeError + expect_error(space.newdict()) # TypeError From dan at codespeak.net Wed Dec 8 20:40:08 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 8 Dec 2010 20:40:08 +0100 (CET) Subject: [pypy-svn] r79911 - pypy/branch/psycopg2compatibility/pypy/module/cpyext/test Message-ID: <20101208194008.8FE28282BE0@codespeak.net> Author: dan Date: Wed Dec 8 20:40:06 2010 New Revision: 79911 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Log: Refactor test to use self.raises from BaseApiTest Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/test/test_sequence.py Wed Dec 8 20:40:06 2010 @@ -72,22 +72,18 @@ assert exc.value.match(space, space.w_StopIteration) def test_setitem(self, space, api): - def expect_error(w_o, i=0, v=42, w_err=space.w_TypeError): - value = space.wrap(v) - result = api.PySequence_SetItem(w_o, i, value) - assert result == -1 - assert api.PyErr_Occurred() - assert api.PyErr_ExceptionMatches(w_err) - api.PyErr_Clear() + w_value = space.wrap(42) l = api.PyList_New(1) - w_value = space.wrap(42) result = api.PySequence_SetItem(l, 0, w_value) assert result != -1 assert space.eq_w(space.getitem(l, space.wrap(0)), w_value) - expect_error(l, 3, - w_err=space.w_IndexError) + self.raises(space, api, IndexError, api.PySequence_SetItem, + l, 3, w_value) + + self.raises(space, api, TypeError, api.PySequence_SetItem, + api.PyTuple_New(1), 0, w_value) - expect_error(api.PyTuple_New(1)) # TypeError - expect_error(space.newdict()) # TypeError + self.raises(space, api, TypeError, api.PySequence_SetItem, + space.newdict(), 0, w_value) From dan at codespeak.net Wed Dec 8 20:49:08 2010 From: dan at codespeak.net (dan at codespeak.net) Date: Wed, 8 Dec 2010 20:49:08 +0100 (CET) Subject: [pypy-svn] r79912 - pypy/branch/psycopg2compatibility/pypy/module/cpyext Message-ID: <20101208194908.D8CF7282BE0@codespeak.net> Author: dan Date: Wed Dec 8 20:49:07 2010 New Revision: 79912 Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py Log: Formatted TypeError message to look like CPython. Modified: pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/psycopg2compatibility/pypy/module/cpyext/sequence.py Wed Dec 8 20:49:07 2010 @@ -1,5 +1,5 @@ -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, operationerrfmt from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t) from pypy.module.cpyext.pyobject import PyObject, borrow_from @@ -139,6 +139,7 @@ changes in your code for properly supporting 64-bit systems.""" if PyDict_Check(space, w_o) or not PySequence_Check(space, w_o): - raise OperationError(space.w_TypeError, "object doesn't support sequence assignment") # FIXME: format like CPython + raise operationerrfmt(space.w_TypeError, "'%s' object does not support item assignment", + space.str_w(space.repr(space.type(w_o)))) # FIXME: looks like lisp... space.setitem(w_o, space.wrap(i), w_v) return 0 From afa at codespeak.net Wed Dec 8 23:25:19 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Wed, 8 Dec 2010 23:25:19 +0100 (CET) Subject: [pypy-svn] r79913 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101208222519.42F6B282BE7@codespeak.net> Author: afa Date: Wed Dec 8 23:25:15 2010 New Revision: 79913 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_code.py Log: *really* call gc_collect Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_code.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_code.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_code.py Wed Dec 8 23:25:15 2010 @@ -134,7 +134,7 @@ coderef = weakref.ref(f.__code__, callback) self.assertTrue(bool(coderef())) del f - test_support.collect() + test_support.gc_collect() self.assertFalse(bool(coderef())) self.assertTrue(self.called) From afa at codespeak.net Thu Dec 9 00:39:59 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 9 Dec 2010 00:39:59 +0100 (CET) Subject: [pypy-svn] r79914 - pypy/branch/fast-forward/pypy/module/__builtin__ Message-ID: <20101208233959.61682282BE7@codespeak.net> Author: afa Date: Thu Dec 9 00:39:55 2010 New Revision: 79914 Modified: pypy/branch/fast-forward/pypy/module/__builtin__/interp_memoryview.py (props changed) Log: fixeol From antocuni at codespeak.net Thu Dec 9 09:13:12 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 9 Dec 2010 09:13:12 +0100 (CET) Subject: [pypy-svn] r79915 - in pypy/branch/jitypes2: . lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/test/output lib_pypy/_ctypes lib_pypy/pypy_test pypy pypy/annotation pypy/config pypy/doc/config pypy/interpreter pypy/interpreter/pyparser pypy/interpreter/pyparser/test pypy/interpreter/test pypy/jit/backend/llsupport pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/optimizeopt pypy/jit/metainterp/test pypy/module/_lsprof pypy/module/_lsprof/test pypy/module/_stackless pypy/module/array/benchmark pypy/module/array/test pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/src pypy/module/cpyext/test pypy/module/exceptions pypy/module/imp pypy/module/pypyjit/test pypy/module/sys pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/test pypy/rpython/lltypesystem pypy/rpython/memory/gc pypy/rpython/memory/gctransform pypy/rpython/tool pypy/tool pypy/tool/release pypy/translator/goal pypy/translator/platform Message-ID: <20101209081312.99C3F282B9D@codespeak.net> Author: antocuni Date: Thu Dec 9 09:13:06 2010 New Revision: 79915 Added: pypy/branch/jitypes2/pypy/doc/config/translation.jit_ffi.txt - copied unchanged from r79914, pypy/trunk/pypy/doc/config/translation.jit_ffi.txt pypy/branch/jitypes2/pypy/module/cpyext/include/fileobject.h - copied unchanged from r79914, pypy/trunk/pypy/module/cpyext/include/fileobject.h pypy/branch/jitypes2/pypy/module/cpyext/pypyintf.py - copied unchanged from r79914, pypy/trunk/pypy/module/cpyext/pypyintf.py Modified: pypy/branch/jitypes2/ (props changed) pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/unixccompiler.py pypy/branch/jitypes2/lib-python/modified-2.5.2/test/output/test_cProfile pypy/branch/jitypes2/lib_pypy/_ctypes/function.py pypy/branch/jitypes2/lib_pypy/pypy_test/test_hashlib.py pypy/branch/jitypes2/pypy/ (props changed) pypy/branch/jitypes2/pypy/annotation/binaryop.py pypy/branch/jitypes2/pypy/config/pypyoption.py pypy/branch/jitypes2/pypy/config/translationoption.py pypy/branch/jitypes2/pypy/interpreter/baseobjspace.py pypy/branch/jitypes2/pypy/interpreter/executioncontext.py pypy/branch/jitypes2/pypy/interpreter/mixedmodule.py pypy/branch/jitypes2/pypy/interpreter/pyparser/error.py pypy/branch/jitypes2/pypy/interpreter/pyparser/pytokenizer.py pypy/branch/jitypes2/pypy/interpreter/pyparser/test/test_pyparse.py pypy/branch/jitypes2/pypy/interpreter/test/test_executioncontext.py pypy/branch/jitypes2/pypy/interpreter/test/test_function.py pypy/branch/jitypes2/pypy/jit/backend/llsupport/llmodel.py pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_runner.py pypy/branch/jitypes2/pypy/jit/codewriter/regalloc.py pypy/branch/jitypes2/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/jitypes2/pypy/jit/metainterp/optimizefindnode.py pypy/branch/jitypes2/pypy/jit/metainterp/optimizeopt/__init__.py pypy/branch/jitypes2/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/branch/jitypes2/pypy/jit/metainterp/pyjitpl.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_compile.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_fficall.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizefficall.py pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jitypes2/pypy/jit/metainterp/warmspot.py pypy/branch/jitypes2/pypy/module/_lsprof/interp_lsprof.py pypy/branch/jitypes2/pypy/module/_lsprof/test/test_cprofile.py pypy/branch/jitypes2/pypy/module/_stackless/interp_coroutine.py pypy/branch/jitypes2/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/jitypes2/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/jitypes2/pypy/module/array/test/test_array.py pypy/branch/jitypes2/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/jitypes2/pypy/module/cpyext/__init__.py pypy/branch/jitypes2/pypy/module/cpyext/api.py pypy/branch/jitypes2/pypy/module/cpyext/include/Python.h pypy/branch/jitypes2/pypy/module/cpyext/include/tupleobject.h pypy/branch/jitypes2/pypy/module/cpyext/intobject.py pypy/branch/jitypes2/pypy/module/cpyext/object.py pypy/branch/jitypes2/pypy/module/cpyext/pyerrors.py pypy/branch/jitypes2/pypy/module/cpyext/pythonrun.py pypy/branch/jitypes2/pypy/module/cpyext/slotdefs.py pypy/branch/jitypes2/pypy/module/cpyext/src/getargs.c pypy/branch/jitypes2/pypy/module/cpyext/state.py pypy/branch/jitypes2/pypy/module/cpyext/stubs.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_cpyext.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_intobject.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_object.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_pyerrors.py pypy/branch/jitypes2/pypy/module/cpyext/test/test_tupleobject.py pypy/branch/jitypes2/pypy/module/cpyext/tupleobject.py pypy/branch/jitypes2/pypy/module/exceptions/interp_exceptions.py pypy/branch/jitypes2/pypy/module/imp/importing.py pypy/branch/jitypes2/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/jitypes2/pypy/module/sys/vm.py pypy/branch/jitypes2/pypy/objspace/std/complexobject.py pypy/branch/jitypes2/pypy/objspace/std/complextype.py pypy/branch/jitypes2/pypy/objspace/std/floattype.py pypy/branch/jitypes2/pypy/objspace/std/longobject.py pypy/branch/jitypes2/pypy/objspace/std/strutil.py pypy/branch/jitypes2/pypy/objspace/std/test/test_complexobject.py pypy/branch/jitypes2/pypy/objspace/std/test/test_strutil.py pypy/branch/jitypes2/pypy/rlib/rerased.py (props changed) pypy/branch/jitypes2/pypy/rlib/test/test_rerased.py (props changed) pypy/branch/jitypes2/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/jitypes2/pypy/rpython/memory/gc/base.py pypy/branch/jitypes2/pypy/rpython/memory/gc/minimark.py pypy/branch/jitypes2/pypy/rpython/memory/gctransform/framework.py pypy/branch/jitypes2/pypy/rpython/tool/rffi_platform.py pypy/branch/jitypes2/pypy/tool/error.py pypy/branch/jitypes2/pypy/tool/release/package.py pypy/branch/jitypes2/pypy/translator/goal/app_main.py pypy/branch/jitypes2/pypy/translator/platform/darwin.py Log: merge from trunk: svn merge svn+ssh://codespeak.net/svn/pypy/trunk -r79770:HEAD . Modified: pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/unixccompiler.py ============================================================================== --- pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/unixccompiler.py (original) +++ pypy/branch/jitypes2/lib-python/modified-2.5.2/distutils/unixccompiler.py Thu Dec 9 09:13:06 2010 @@ -121,7 +121,22 @@ } if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() executables['ranlib'] = ["ranlib"] + executables['linker_so'] += ['-undefined', 'dynamic_lookup'] + + for k, v in executables.iteritems(): + if v and v[0] == 'cc': + v += ['-arch', arch] + # Needed for the filename generation methods provided by the base # class, CCompiler. NB. whoever instantiates/uses a particular Modified: pypy/branch/jitypes2/lib-python/modified-2.5.2/test/output/test_cProfile ============================================================================== --- pypy/branch/jitypes2/lib-python/modified-2.5.2/test/output/test_cProfile (original) +++ pypy/branch/jitypes2/lib-python/modified-2.5.2/test/output/test_cProfile Thu Dec 9 09:13:06 2010 @@ -14,66 +14,66 @@ 4 0.116 0.029 0.120 0.030 test_cProfile.py:78(helper1) 2 0.000 0.000 0.140 0.070 test_cProfile.py:89(helper2_indirect) 8 0.312 0.039 0.400 0.050 test_cProfile.py:93(helper2) - 4 0.000 0.000 0.000 0.000 {append} - 1 0.000 0.000 0.000 0.000 {disable} 12 0.000 0.000 0.012 0.001 {hasattr} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} + 1 0.000 0.000 0.000 0.000 {method 'disable' of 'Profile' objects} 8 0.000 0.000 0.000 0.000 {range} 4 0.000 0.000 0.000 0.000 {sys.exc_info} Ordered by: standard name -Function called... - ncalls tottime cumtime -:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) -test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) - 8 0.000 0.000 {range} -test_cProfile.py:115(__getattr__) -> -test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) - 2 0.040 0.600 test_cProfile.py:60(helper) -test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 20 0.020 0.020 test_cProfile.py:53(mul) -test_cProfile.py:53(mul) -> -test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) - 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) - 6 0.234 0.300 test_cProfile.py:93(helper2) -test_cProfile.py:78(helper1) -> 4 0.000 0.000 {append} - 4 0.000 0.004 {hasattr} - 4 0.000 0.000 {sys.exc_info} -test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) - 2 0.078 0.100 test_cProfile.py:93(helper2) -test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) - 8 0.000 0.008 {hasattr} -{append} -> -{disable} -> -{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) -{range} -> -{sys.exc_info} -> +Function called... + ncalls tottime cumtime +:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) +test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) + 8 0.000 0.000 {range} +test_cProfile.py:115(__getattr__) -> +test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) + 2 0.040 0.600 test_cProfile.py:60(helper) +test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 20 0.020 0.020 test_cProfile.py:53(mul) +test_cProfile.py:53(mul) -> +test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) + 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) + 6 0.234 0.300 test_cProfile.py:93(helper2) +test_cProfile.py:78(helper1) -> 4 0.000 0.004 {hasattr} + 4 0.000 0.000 {method 'append' of 'list' objects} + 4 0.000 0.000 {sys.exc_info} +test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) + 2 0.078 0.100 test_cProfile.py:93(helper2) +test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) + 8 0.000 0.008 {hasattr} +{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) +{method 'append' of 'list' objects} -> +{method 'disable' of 'Profile' objects} -> +{range} -> +{sys.exc_info} -> Ordered by: standard name -Function was called by... - ncalls tottime cumtime -:1() <- -test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) -test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) - 12 0.012 0.012 {hasattr} -test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() -test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) - 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) -test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) -test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) -test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) -test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) -test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) - 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) -{append} <- 4 0.000 0.000 test_cProfile.py:78(helper1) -{disable} <- -{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) - 8 0.000 0.008 test_cProfile.py:93(helper2) -{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) -{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +Function was called by... + ncalls tottime cumtime +:1() <- +test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) +test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) + 12 0.012 0.012 {hasattr} +test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() +test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) + 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) +test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) +test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) +test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) +test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) +test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) + 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) +{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) + 8 0.000 0.008 test_cProfile.py:93(helper2) +{method 'append' of 'list' objects} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +{method 'disable' of 'Profile' objects} <- +{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) +{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Thu Dec 9 09:13:06 2010 @@ -199,9 +199,8 @@ restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype, thisarg) result = funcptr(*newargs) - if restype and restype._ffishape == 'u': - # XXX: maybe it's a job of _ffi? - result = unichr(result) + result = self._wrap_result(restype, result) + # ## resbuffer = funcptr(*[arg._get_buffer_for_param()._buffer ## for arg in args]) ## result = self._build_result(restype, resbuffer, argtypes, args) @@ -383,6 +382,8 @@ wrapped_args.append(wrapped) return wrapped_args + + # XXX: maybe the following two methods should be done inside _ffi? def _unwrap_args(self, argtypes, args): """ Convert from ctypes high-level values to low-level values suitables to @@ -399,13 +400,31 @@ elif argtype._ffishape == 'P': value = arg._buffer.buffer if value > sys.maxint: - # XXX: simulate overflow so that _ffi receive and int, not a long + # XXX: workaround for old versions of pypy-c, as soon as + # translation works again we can remove it value = (-sys.maxint-1)*2 + value else: value = arg.value newargs.append(value) return newargs + def _wrap_result(self, restype, result): + """ + Convert from low-level repr of the result to the high-level python + one: e.g., if the restype is a pointer 0 is converted to None, and + for chars we convert the int value with chr, etc. + """ + if not restype: + return None + elif restype._ffishape == 'u': + result = unichr(result) + elif restype._ffishape == 'P': + if result == 0: + result = None + else: + assert False, 'TODO' + return result + def _build_result(self, restype, resbuffer, argtypes, argsandobjs): """Build the function result: If there is no OUT parameter, return the actual function result Modified: pypy/branch/jitypes2/lib_pypy/pypy_test/test_hashlib.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/pypy_test/test_hashlib.py (original) +++ pypy/branch/jitypes2/lib_pypy/pypy_test/test_hashlib.py Thu Dec 9 09:13:06 2010 @@ -41,7 +41,8 @@ # also test the pure Python implementation modname, constructor = pure_python_version[name].split('.') - builder = getattr(__import__(modname), constructor) + mod = __import__('lib_pypy.' + modname, None, None, ['__doc__']) + builder = getattr(mod, constructor) h = builder('') assert h.digest_size == expected_size assert h.digestsize == expected_size Modified: pypy/branch/jitypes2/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/jitypes2/pypy/annotation/binaryop.py (original) +++ pypy/branch/jitypes2/pypy/annotation/binaryop.py Thu Dec 9 09:13:06 2010 @@ -861,7 +861,7 @@ def getitem((p, obj)): assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - def setitem((p, obj)): + def setitem((p, obj), s_value): assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) class __extend__(pairtype(SomeObject, SomePtr)): Modified: pypy/branch/jitypes2/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/jitypes2/pypy/config/pypyoption.py (original) +++ pypy/branch/jitypes2/pypy/config/pypyoption.py Thu Dec 9 09:13:06 2010 @@ -76,6 +76,7 @@ "_rawffi": [("objspace.usemodules.struct", True)], "cpyext": [("translation.secondaryentrypoints", "cpyext"), ("translation.shared", sys.platform == "win32")], + "_ffi": [("translation.jit_ffi", True)], } module_import_dependencies = { Modified: pypy/branch/jitypes2/pypy/config/translationoption.py ============================================================================== --- pypy/branch/jitypes2/pypy/config/translationoption.py (original) +++ pypy/branch/jitypes2/pypy/config/translationoption.py Thu Dec 9 09:13:06 2010 @@ -117,6 +117,7 @@ ChoiceOption("jit_profiler", "integrate profiler support into the JIT", ["off", "oprofile"], default="off"), + BoolOption("jit_ffi", "optimize libffi calls", default=False), # misc BoolOption("verbose", "Print extra information", default=False), Modified: pypy/branch/jitypes2/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/baseobjspace.py Thu Dec 9 09:13:06 2010 @@ -862,14 +862,14 @@ def call_args_and_c_profile(self, frame, w_func, args): ec = self.getexecutioncontext() - ec.c_call_trace(frame, w_func) + ec.c_call_trace(frame, w_func, args) try: w_res = self.call_args(w_func, args) except OperationError, e: w_value = e.get_w_value(self) ec.c_exception_trace(frame, w_value) raise - ec.c_return_trace(frame, w_func) + ec.c_return_trace(frame, w_func, args) return w_res def call_method(self, w_obj, methname, *arg_w): Modified: pypy/branch/jitypes2/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/executioncontext.py Thu Dec 9 09:13:06 2010 @@ -27,7 +27,6 @@ def __init__(self, space): self.space = space self.topframeref = jit.vref_None - self.framestackdepth = 0 # tracing: space.frame_trace_action.fire() must be called to ensure # that tracing occurs whenever self.w_tracefunc or self.is_tracing # is modified. @@ -54,9 +53,6 @@ return frame def enter(self, frame): - if self.framestackdepth > self.space.sys.recursionlimit: - raise self.space.prebuilt_recursion_error - self.framestackdepth += 1 frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) @@ -66,7 +62,6 @@ self._trace(frame, 'leaveframe', self.space.w_None) finally: self.topframeref = frame.f_backref - self.framestackdepth -= 1 jit.virtual_ref_finish(frame) if self.w_tracefunc is not None and not frame.hide(): @@ -80,7 +75,6 @@ def __init__(self): self.topframe = None - self.framestackdepth = 0 self.w_tracefunc = None self.profilefunc = None self.w_profilefuncarg = None @@ -88,7 +82,6 @@ def enter(self, ec): ec.topframeref = jit.non_virtual_ref(self.topframe) - ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc ec.w_profilefuncarg = self.w_profilefuncarg @@ -97,7 +90,6 @@ def leave(self, ec): self.topframe = ec.gettopframe() - self.framestackdepth = ec.framestackdepth self.w_tracefunc = ec.w_tracefunc self.profilefunc = ec.profilefunc self.w_profilefuncarg = ec.w_profilefuncarg @@ -105,7 +97,6 @@ def clear_framestack(self): self.topframe = None - self.framestackdepth = 0 # the following interface is for pickling and unpickling def getstate(self, space): @@ -121,33 +112,41 @@ self.topframe = space.interp_w(PyFrame, frames_w[-1]) else: self.topframe = None - self.framestackdepth = len(frames_w) def getframestack(self): - index = self.framestackdepth - lst = [None] * index + lst = [] f = self.topframe - while index > 0: - index -= 1 - lst[index] = f + while f is not None: + lst.append(f) f = f.f_backref() - assert f is None + lst.reverse() return lst # coroutine: I think this is all, folks! - def c_call_trace(self, frame, w_func): + def c_call_trace(self, frame, w_func, args=None): "Profile the call of a builtin function" - if self.profilefunc is None: - frame.is_being_profiled = False - else: - self._trace(frame, 'c_call', w_func) + self._c_call_return_trace(frame, w_func, args, 'c_call') - def c_return_trace(self, frame, w_retval): + def c_return_trace(self, frame, w_func, args=None): "Profile the return from a builtin function" + self._c_call_return_trace(frame, w_func, args, 'c_return') + + def _c_call_return_trace(self, frame, w_func, args, event): if self.profilefunc is None: frame.is_being_profiled = False else: - self._trace(frame, 'c_return', w_retval) + # undo the effect of the CALL_METHOD bytecode, which would be + # that even on a built-in method call like '[].append()', + # w_func is actually the unbound function 'append'. + from pypy.interpreter.function import FunctionWithFixedCode + if isinstance(w_func, FunctionWithFixedCode) and args is not None: + w_firstarg = args.firstarg() + if w_firstarg is not None: + from pypy.interpreter.function import descr_function_get + w_func = descr_function_get(self.space, w_func, w_firstarg, + self.space.type(w_firstarg)) + # + self._trace(frame, event, w_func) def c_exception_trace(self, frame, w_exc): "Profile function called upon OperationError." Modified: pypy/branch/jitypes2/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/mixedmodule.py Thu Dec 9 09:13:06 2010 @@ -13,6 +13,10 @@ applevel_name = None expose__file__attribute = True + + # The following attribute is None as long as the module has not been + # imported yet, and when it has been, it is mod.__dict__.items() just + # after startup(). w_initialdict = None def __init__(self, space, w_name): @@ -26,8 +30,14 @@ """This is called each time the module is imported or reloaded """ if self.w_initialdict is not None: + # the module was already imported. Refresh its content with + # the saved dict, as done with built-in and extension modules + # on CPython. space.call_method(self.w_dict, 'update', self.w_initialdict) - Module.init(self, space) + else: + Module.init(self, space) + if not self.lazy and self.w_initialdict is None: + self.w_initialdict = space.call_method(self.w_dict, 'items') def get_applevel_name(cls): """ NOT_RPYTHON """ @@ -96,6 +106,7 @@ def _freeze_(self): self.getdict() + self.w_initialdict = None self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant Modified: pypy/branch/jitypes2/pypy/interpreter/pyparser/error.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/pyparser/error.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/pyparser/error.py Thu Dec 9 09:13:06 2010 @@ -2,19 +2,22 @@ class SyntaxError(Exception): """Base class for exceptions raised by the parser.""" - def __init__(self, msg, lineno=0, offset=0, text=None, filename=None): + def __init__(self, msg, lineno=0, offset=0, text=None, filename=None, + lastlineno=0): self.msg = msg self.lineno = lineno self.offset = offset self.text = text self.filename = filename + self.lastlineno = lastlineno def wrap_info(self, space): return space.newtuple([space.wrap(self.msg), space.newtuple([space.wrap(self.filename), space.wrap(self.lineno), space.wrap(self.offset), - space.wrap(self.text)])]) + space.wrap(self.text), + space.wrap(self.lastlineno)])]) def __str__(self): return "%s at pos (%d, %d) in %r" % (self.__class__.__name__, @@ -33,8 +36,9 @@ class TokenError(SyntaxError): - def __init__(self, msg, line, lineno, column, tokens): - SyntaxError.__init__(self, msg, lineno, column, line) + def __init__(self, msg, line, lineno, column, tokens, lastlineno=0): + SyntaxError.__init__(self, msg, lineno, column, line, + lastlineno=lastlineno) self.tokens = tokens class TokenIndentationError(IndentationError): Modified: pypy/branch/jitypes2/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/jitypes2/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/jitypes2/pypy/interpreter/pyparser/pytokenizer.py Thu Dec 9 09:13:06 2010 @@ -78,6 +78,7 @@ contline = None indents = [0] last_comment = '' + parenlevstart = (0, 0, "") # make the annotator happy endDFA = automata.DFA([], []) @@ -85,7 +86,7 @@ line = '' pos = 0 lines.append("") - strstart = (0, 0) + strstart = (0, 0, "") for line in lines: lnum = lnum + 1 pos, max = 0, len(line) @@ -93,7 +94,8 @@ if contstr: if not line: raise TokenError("EOF while scanning triple-quoted string", - line, lnum-1, 0, token_list) + strstart[2], strstart[0], strstart[1]+1, + token_list, lnum) endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch @@ -146,6 +148,10 @@ else: # continued statement if not line: + if parenlev > 0: + lnum1, start1, line1 = parenlevstart + raise TokenError("parenthesis is never closed", line1, + lnum1, start1 + 1, token_list, lnum) raise TokenError("EOF in multi-line statement", line, lnum, 0, token_list) continued = 0 @@ -187,7 +193,7 @@ token_list.append(tok) last_comment = '' else: - strstart = (lnum, start) + strstart = (lnum, start, line) contstr = line[start:] contline = line break @@ -195,7 +201,7 @@ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string - strstart = (lnum, start) + strstart = (lnum, start, line) endDFA = (endDFAs[initial] or endDFAs[token[1]] or endDFAs[token[2]]) contstr, needcont = line[start:], 1 @@ -212,6 +218,8 @@ continued = 1 else: if initial in '([{': + if parenlev == 0: + parenlevstart = (lnum, start, line) parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 @@ -230,7 +238,7 @@ start = pos if start" - assert repr(A().f).startswith(">") + assert repr(A().f).startswith(">") class B: def f(self): pass assert repr(B.f) == "" assert repr(B().f).startswith(">") + assert repr(A().f).endswith(">>") def test_method_call(self): @@ -487,14 +487,21 @@ def f(): pass raises(TypeError, new.instancemethod, f, None) + def test_empty_arg_kwarg_call(self): + def f(): + pass -class TestMethod: + raises(TypeError, lambda: f(*0)) + raises(TypeError, lambda: f(**0)) + + +class TestMethod: def setup_method(self, method): def c(self, bar): return bar code = PyCode._from_code(self.space, c.func_code) self.fn = Function(self.space, code, self.space.newdict()) - + def test_get(self): space = self.space w_meth = descr_function_get(space, self.fn, space.wrap(5), space.type(space.wrap(5))) @@ -552,7 +559,7 @@ def test_call_function(self): space = self.space - + d = {} for i in range(10): args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")" @@ -574,14 +581,14 @@ code.funcrun = bomb code.funcrun_obj = bomb - args_w = map(space.wrap, range(i)) + args_w = map(space.wrap, range(i)) w_res = space.call_function(fn, *args_w) check = space.is_true(space.eq(w_res, space.wrap(res))) assert check def test_flatcall(self): space = self.space - + def f(a): return a code = PyCode._from_code(self.space, f.func_code) @@ -608,7 +615,7 @@ def test_flatcall_method(self): space = self.space - + def f(self, a): return a code = PyCode._from_code(self.space, f.func_code) @@ -636,7 +643,7 @@ def test_flatcall_default_arg(self): space = self.space - + def f(a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -665,7 +672,7 @@ def test_flatcall_default_arg_method(self): space = self.space - + def f(self, a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -688,7 +695,7 @@ y = A().m(x) b = A().m z = b(x) - return y+10*z + return y+10*z """) assert space.eq_w(w_res, space.wrap(44)) Modified: pypy/branch/jitypes2/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/llsupport/llmodel.py Thu Dec 9 09:13:06 2010 @@ -17,7 +17,6 @@ from pypy.jit.backend.llsupport.descr import get_call_descr from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr -from pypy.jit.backend.llsupport.ffisupport import get_call_descr_dynamic from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -247,7 +246,9 @@ return get_call_descr(self.gc_ll_descr, ARGS, RESULT, extrainfo) def calldescrof_dynamic(self, ffi_args, ffi_result, extrainfo=None): - return get_call_descr_dynamic(ffi_args, ffi_result, extrainfo) + from pypy.jit.backend.llsupport import ffisupport + return ffisupport.get_call_descr_dynamic(ffi_args, ffi_result, + extrainfo) def get_overflow_error(self): ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable) Modified: pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_runner.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_runner.py (original) +++ pypy/branch/jitypes2/pypy/jit/backend/x86/test/test_runner.py Thu Dec 9 09:13:06 2010 @@ -82,7 +82,7 @@ # relative addressing to work properly. addr = rffi.cast(lltype.Signed, addr) - self.cpu.assembler.setup() + self.cpu.assembler.setup_once() self.cpu.assembler.malloc_func_addr = addr ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0] Modified: pypy/branch/jitypes2/pypy/jit/codewriter/regalloc.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/codewriter/regalloc.py (original) +++ pypy/branch/jitypes2/pypy/jit/codewriter/regalloc.py Thu Dec 9 09:13:06 2010 @@ -36,7 +36,7 @@ if isinstance(v1, Variable): die_at[v1] = i if op.result is not None: - die_at[op.result] = i + die_at[op.result] = i + 1 if isinstance(block.exitswitch, tuple): for x in block.exitswitch: die_at.pop(x, None) Modified: pypy/branch/jitypes2/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/jitypes2/pypy/jit/codewriter/test/test_regalloc.py Thu Dec 9 09:13:06 2010 @@ -281,22 +281,22 @@ # this used to produce bogus code, containing these two # lines in the following broken order: # last_exc_value -> %r0 - # ref_copy %r0 -> %r2 -- but expect to read the old value of %r0! + # ref_copy %r0 -> %r1 -- but expect to read the old value of %r0! self.check_assembler(graph, """ residual_call_r_r $<* fn bar>, , R[%r0] -> %r1 -live- - residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r2 + residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r1 -live- catch_exception L1 - ref_return %r2 + ref_return %r1 --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 - ref_copy %r0 -> %r2 + ref_copy %r0 -> %r1 last_exc_value -> %r0 residual_call_r_r $<* fn foo>, , R[%r0] -> %r0 -live- - ref_return %r2 + ref_return %r1 --- L2: reraise Modified: pypy/branch/jitypes2/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/optimizefindnode.py Thu Dec 9 09:13:06 2010 @@ -128,6 +128,7 @@ assert isinstance(constbox, Const) node = InstanceNode() node.unique = UNIQUE_NO + node.escaped = True node.knownvaluebox = constbox self.nodes[box] = node return node Modified: pypy/branch/jitypes2/pypy/jit/metainterp/optimizeopt/__init__.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/optimizeopt/__init__.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/optimizeopt/__init__.py Thu Dec 9 09:13:06 2010 @@ -3,7 +3,6 @@ from pypy.jit.metainterp.optimizeopt.intbounds import OptIntBounds from pypy.jit.metainterp.optimizeopt.virtualize import OptVirtualize from pypy.jit.metainterp.optimizeopt.heap import OptHeap -from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall from pypy.jit.metainterp.optimizeopt.string import OptString def optimize_loop_1(metainterp_sd, loop, virtuals=True): @@ -17,6 +16,10 @@ OptVirtualize(), OptString(), OptHeap(), + ] + if metainterp_sd.jit_ffi: + from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall + optimizations = optimizations + [ OptFfiCall(), ] optimizer = Optimizer(metainterp_sd, loop, optimizations, virtuals) Modified: pypy/branch/jitypes2/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/pyjitpl.py Thu Dec 9 09:13:06 2010 @@ -1216,7 +1216,8 @@ logger_ops = None def __init__(self, cpu, options, - ProfilerClass=EmptyProfiler, warmrunnerdesc=None): + ProfilerClass=EmptyProfiler, warmrunnerdesc=None, + jit_ffi=True): self.cpu = cpu self.stats = self.cpu.stats self.options = options @@ -1226,6 +1227,7 @@ self.profiler = ProfilerClass() self.profiler.cpu = cpu self.warmrunnerdesc = warmrunnerdesc + self.jit_ffi = jit_ffi backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_compile.py Thu Dec 9 09:13:06 2010 @@ -54,6 +54,7 @@ stats = Stats() profiler = jitprof.EmptyProfiler() warmrunnerdesc = None + jit_ffi = False def log(self, msg, event_kind=None): pass Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_fficall.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_fficall.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_fficall.py Thu Dec 9 09:13:06 2010 @@ -44,6 +44,6 @@ n += 1 return res # - res = self.meta_interp(f, [0]) + res = self.meta_interp(f, [0], jit_ffi=True) return res Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizefficall.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizefficall.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizefficall.py Thu Dec 9 09:13:06 2010 @@ -31,6 +31,7 @@ class TestFfiCall(BaseTestOptimizeOpt, LLtypeMixin): + jit_ffi = True class namespace: cpu = LLtypeMixin.cpu Modified: pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/test/test_optimizeopt.py Thu Dec 9 09:13:06 2010 @@ -28,11 +28,12 @@ class FakeMetaInterpStaticData(object): - def __init__(self, cpu): + def __init__(self, cpu, jit_ffi=False): self.cpu = cpu self.profiler = EmptyProfiler() self.options = Fake() self.globaldata = Fake() + self.jit_ffi = jit_ffi def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr @@ -227,6 +228,7 @@ return sorted(boxes, key=lambda box: _kind2count[box.type]) class BaseTestOptimizeOpt(BaseTest): + jit_ffi = False def invent_fail_descr(self, fail_args): if fail_args is None: @@ -261,7 +263,7 @@ loop.token.specnodes = self.unpack_specnodes(spectext) # self.loop = loop - metainterp_sd = FakeMetaInterpStaticData(self.cpu) + metainterp_sd = FakeMetaInterpStaticData(self.cpu, self.jit_ffi) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo if hasattr(self, 'callinfocollection'): @@ -1954,6 +1956,15 @@ """ self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)', expected) + def test_bug_4(self): + ops = """ + [p9] + p30 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) + jump(p30) + """ + self.optimize_loop(ops, 'Not', ops) + def test_invalid_loop_1(self): ops = """ [p1] Modified: pypy/branch/jitypes2/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jitypes2/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jitypes2/pypy/jit/metainterp/warmspot.py Thu Dec 9 09:13:06 2010 @@ -144,7 +144,8 @@ class WarmRunnerDesc(object): def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, - optimizer=None, ProfilerClass=EmptyProfiler, **kwds): + optimizer=None, ProfilerClass=EmptyProfiler, + jit_ffi=None, **kwds): pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) self.memory_manager = memmgr.MemoryManager() @@ -162,7 +163,7 @@ elif self.opt.listops: self.prejit_optimizations_minimal_inline(policy, graphs) - self.build_meta_interp(ProfilerClass) + self.build_meta_interp(ProfilerClass, jit_ffi) self.make_args_specifications() # from pypy.jit.metainterp.virtualref import VirtualRefInfo @@ -280,11 +281,14 @@ translate_support_code, gcdescr=self.gcdescr) self.cpu = cpu - def build_meta_interp(self, ProfilerClass): + def build_meta_interp(self, ProfilerClass, jit_ffi=None): + if jit_ffi is None: + jit_ffi = self.translator.config.translation.jit_ffi self.metainterp_sd = MetaInterpStaticData(self.cpu, self.opt, ProfilerClass=ProfilerClass, - warmrunnerdesc=self) + warmrunnerdesc=self, + jit_ffi=jit_ffi) def make_virtualizable_infos(self): vinfos = {} Modified: pypy/branch/jitypes2/pypy/module/_lsprof/interp_lsprof.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_lsprof/interp_lsprof.py (original) +++ pypy/branch/jitypes2/pypy/module/_lsprof/interp_lsprof.py Thu Dec 9 09:13:06 2010 @@ -163,8 +163,11 @@ if isinstance(w_arg, Method): w_function = w_arg.w_function class_name = w_arg.w_class.getname(space, '?') - assert isinstance(w_function, Function) - return "{method '%s' of '%s' objects}" % (w_function.name, class_name) + if isinstance(w_function, Function): + name = w_function.name + else: + name = '?' + return "{method '%s' of '%s' objects}" % (name, class_name) elif isinstance(w_arg, Function): if w_arg.w_module is None: module = '' @@ -176,7 +179,8 @@ module += '.' return '{%s%s}' % (module, w_arg.name) else: - return '{!!!unknown!!!}' + class_name = space.type(w_arg).getname(space, '?') + return "{'%s' object}" % (class_name,) def lsprof_call(space, w_self, frame, event, w_arg): assert isinstance(w_self, W_Profiler) Modified: pypy/branch/jitypes2/pypy/module/_lsprof/test/test_cprofile.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_lsprof/test/test_cprofile.py (original) +++ pypy/branch/jitypes2/pypy/module/_lsprof/test/test_cprofile.py Thu Dec 9 09:13:06 2010 @@ -2,9 +2,10 @@ from pypy.conftest import gettestobjspace, option class AppTestCProfile(object): + keywords = {} def setup_class(cls): - space = gettestobjspace(usemodules=('_lsprof',)) + space = gettestobjspace(usemodules=('_lsprof',), **cls.keywords) cls.w_expected_output = space.wrap(expected_output) cls.space = space cls.w_file = space.wrap(__file__) @@ -148,6 +149,12 @@ finally: sys.path.pop(0) + +class AppTestWithDifferentBytecodes(AppTestCProfile): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True, + 'objspace.opcodes.CALL_METHOD': True} + + expected_output = {} expected_output['print_stats'] = """\ 126 function calls (106 primitive calls) in 1.000 CPU seconds @@ -165,11 +172,11 @@ 2 0.000 0.000 0.140 0.070 profilee.py:84(helper2_indirect) 8 0.312 0.039 0.400 0.050 profilee.py:88(helper2) 8 0.064 0.008 0.080 0.010 profilee.py:98(subhelper) - 4 0.000 0.000 0.000 0.000 {.*append.*} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} 1 0.000 0.000 0.000 0.000 {.*disable.*} - 12 0.000 0.000 0.012 0.001 {hasattr.*} - 8 0.000 0.000 0.000 0.000 {range.*} - 4 0.000 0.000 0.000 0.000 {sys.exc_info.*} + 12 0.000 0.000 0.012 0.001 {hasattr} + 8 0.000 0.000 0.000 0.000 {range} + 4 0.000 0.000 0.000 0.000 {sys.exc_info} """ Modified: pypy/branch/jitypes2/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/jitypes2/pypy/module/_stackless/interp_coroutine.py Thu Dec 9 09:13:06 2010 @@ -304,16 +304,19 @@ def w_descr__framestack(space, self): assert isinstance(self, AppCoroutine) - index = self.subctx.framestackdepth - if not index: - return space.newtuple([]) - items = [None] * index + counter = 0 f = self.subctx.topframe - while index > 0: - index -= 1 - items[index] = space.wrap(f) + while f is not None: + counter += 1 f = f.f_backref() - assert f is None + items = [None] * counter + f = self.subctx.topframe + while f is not None: + counter -= 1 + assert counter >= 0 + items[counter] = space.wrap(f) + f = f.f_backref() + assert counter == 0 return space.newtuple(items) def makeStaticMethod(module, classname, funcname): Modified: pypy/branch/jitypes2/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/array/test/test_array.py (original) +++ pypy/branch/jitypes2/pypy/module/array/test/test_array.py Thu Dec 9 09:13:06 2010 @@ -64,6 +64,7 @@ raises(TypeError, self.array, tc, None) def test_value_range(self): + import sys values = (-129, 128, -128, 127, 0, 255, -1, 256, -32768, 32767, -32769, 32768, 65535, 65536, -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, @@ -88,7 +89,12 @@ a.append(v) for i, v in enumerate(ok * 2): assert a[i] == v - assert type(a[i]) is pt + assert type(a[i]) is pt or ( + # A special case: we return ints in Array('I') on 64-bits, + # whereas CPython returns longs. The difference is + # probably acceptable. + tc == 'I' and + sys.maxint > 2147483647 and type(a[i]) is int) for v in ok: a[1] = v assert a[0] == ok[0] Modified: pypy/branch/jitypes2/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/__init__.py Thu Dec 9 09:13:06 2010 @@ -69,6 +69,7 @@ import pypy.module.cpyext.weakrefobject import pypy.module.cpyext.funcobject import pypy.module.cpyext.classobject +import pypy.module.cpyext.pypyintf # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/jitypes2/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/api.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/api.py Thu Dec 9 09:13:06 2010 @@ -369,7 +369,7 @@ }.items(): GLOBALS['Py%s_Type#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) - for cpyname in 'Method List Int Long Dict Class'.split(): + for cpyname in 'Method List Int Long Dict Tuple Class'.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } ' 'Py%sObject' % (cpyname, )) build_exported_objects() Modified: pypy/branch/jitypes2/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/include/Python.h Thu Dec 9 09:13:06 2010 @@ -8,6 +8,8 @@ # include # include # include +# include +# include # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_FUNC(RTYPE) RTYPE # define PyAPI_DATA(RTYPE) extern RTYPE @@ -103,6 +105,7 @@ #include "sliceobject.h" #include "datetime.h" #include "pystate.h" +#include "fileobject.h" // XXX This shouldn't be included here #include "structmember.h" @@ -120,4 +123,8 @@ #define PyDoc_STR(str) "" #endif +/* PyPy does not implement --with-fpectl */ +#define PyFPE_START_PROTECT(err_string, leave_stmt) +#define PyFPE_END_PROTECT(v) + #endif Modified: pypy/branch/jitypes2/pypy/module/cpyext/include/tupleobject.h ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/include/tupleobject.h (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/include/tupleobject.h Thu Dec 9 09:13:06 2010 @@ -10,19 +10,9 @@ /* defined in varargswrapper.c */ PyObject * PyTuple_Pack(Py_ssize_t, ...); -typedef struct { - PyObject_HEAD - PyObject **items; - Py_ssize_t size; -} PyTupleObject; +#define PyTuple_SET_ITEM PyTuple_SetItem +#define PyTuple_GET_ITEM PyTuple_GetItem -#define PyTuple_GET_ITEM PyTuple_GetItem - -/* Macro, trading safety for speed */ -#define PyTuple_GET_SIZE(op) (((PyTupleObject *)(op))->size) - -/* Macro, *only* to be used to fill in brand new tuples */ -#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->items[i] = v) #ifdef __cplusplus } Modified: pypy/branch/jitypes2/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/intobject.py Thu Dec 9 09:13:06 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, build_type_checkers, Py_ssize_t) @@ -19,6 +20,9 @@ already one, and then return its value. If there is an error, -1 is returned, and the caller should check PyErr_Occurred() to find out whether there was an error, or whether the value just happened to be -1.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Unsigned, error=-1) @@ -26,6 +30,9 @@ """Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.uint_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Signed, error=CANNOT_FAIL) @@ -39,6 +46,9 @@ PyLongObject, if it is not already one, and then return its value as Py_ssize_t. """ + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(w_obj) # XXX this is wrong on win64 @cpython_api([Py_ssize_t], PyObject) @@ -48,4 +58,3 @@ returned. """ return space.wrap(ival) # XXX this is wrong on win64 - Modified: pypy/branch/jitypes2/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/object.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/object.py Thu Dec 9 09:13:06 2010 @@ -2,12 +2,13 @@ from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, - Py_GE, CONST_STRING, FILEP, fwrite) + Py_GE, CONST_STRING, FILEP, fwrite, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference, get_typedescr, RefcountState) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall +from pypy.module._file.interp_file import W_File from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject from pypy.interpreter.error import OperationError @@ -428,6 +429,34 @@ rffi.free_nonmovingbuffer(data, buf) return 0 +PyFile_Check, PyFile_CheckExact = build_type_checkers("File", W_File) + + at cpython_api([PyObject, rffi.INT_real], PyObject) +def PyFile_GetLine(space, w_obj, n): + """ + Equivalent to p.readline([n]), this function reads one line from the + object p. p may be a file object or any object with a readline() + method. If n is 0, exactly one line is read, regardless of the length of + the line. If n is greater than 0, no more than n bytes will be read + from the file; a partial line can be returned. In both cases, an empty string + is returned if the end of the file is reached immediately. If n is less than + 0, however, one line is read regardless of length, but EOFError is + raised if the end of the file is reached immediately.""" + try: + w_readline = space.getattr(w_obj, space.wrap('readline')) + except OperationError: + raise OperationError( + space.w_TypeError, space.wrap( + "argument must be a file, or have a readline() method.")) + + n = rffi.cast(lltype.Signed, n) + if space.is_true(space.gt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline, space.wrap(n)) + elif space.is_true(space.lt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline) + else: + # XXX Raise EOFError as specified + return space.call_function(w_readline) @cpython_api([CONST_STRING, CONST_STRING], PyObject) def PyFile_FromString(space, filename, mode): """ @@ -437,4 +466,3 @@ w_filename = space.wrap(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) - Modified: pypy/branch/jitypes2/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/pyerrors.py Thu Dec 9 09:13:06 2010 @@ -5,7 +5,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, Py_DecRef, borrow_from) + PyObject, PyObjectP, make_ref, from_ref, Py_DecRef, borrow_from) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from pypy.rlib.rposix import get_errno @@ -80,6 +80,21 @@ Py_DecRef(space, w_value) Py_DecRef(space, w_traceback) + at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) +def PyErr_NormalizeException(space, exc_p, val_p, tb_p): + """Under certain circumstances, the values returned by PyErr_Fetch() below + can be "unnormalized", meaning that *exc is a class object but *val is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance.""" + operr = OperationError(from_ref(space, exc_p[0]), + from_ref(space, val_p[0])) + operr.normalize_exception(space) + Py_DecRef(space, exc_p[0]) + Py_DecRef(space, val_p[0]) + exc_p[0] = make_ref(space, operr.w_type) + val_p[0] = make_ref(space, operr.get_w_value(space)) + @cpython_api([], lltype.Void) def PyErr_BadArgument(space): """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where @@ -114,10 +129,29 @@ function around a system call can write return PyErr_SetFromErrno(type); when the system call returns an error. Return value: always NULL.""" + PyErr_SetFromErrnoWithFilename(space, w_type, + lltype.nullptr(rffi.CCHARP.TO)) + + at cpython_api([PyObject, rffi.CCHARP], PyObject) +def PyErr_SetFromErrnoWithFilename(space, w_type, llfilename): + """Similar to PyErr_SetFromErrno(), with the additional behavior that if + filename is not NULL, it is passed to the constructor of type as a third + parameter. In the case of exceptions such as IOError and OSError, + this is used to define the filename attribute of the exception instance. + Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() msg = os.strerror(errno) - w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + if llfilename: + w_filename = rffi.charp2str(llfilename) + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg), + space.wrap(w_filename)) + else: + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg)) raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) Modified: pypy/branch/jitypes2/pypy/module/cpyext/pythonrun.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/pythonrun.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/pythonrun.py Thu Dec 9 09:13:06 2010 @@ -1,6 +1,16 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.state import State @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def Py_IsInitialized(space): return 1 + + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) +def Py_GetProgramName(space): + """ + Return the program name set with Py_SetProgramName(), or the default. + The returned string points into static storage; the caller should not modify its + value.""" + return space.fromcache(State).get_programname() + Modified: pypy/branch/jitypes2/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/slotdefs.py Thu Dec 9 09:13:06 2010 @@ -28,14 +28,14 @@ def check_num_args(space, ob, n): from pypy.module.cpyext.tupleobject import PyTuple_CheckExact, \ - _PyTuple_Size_Fast + PyTuple_GET_SIZE if not PyTuple_CheckExact(space, ob): raise OperationError(space.w_SystemError, space.wrap("PyArg_UnpackTuple() argument list is not a tuple")) - if n == _PyTuple_Size_Fast(space, ob): + if n == PyTuple_GET_SIZE(space, ob): return raise operationerrfmt(space.w_TypeError, - "expected %d arguments, got %d", n, _PyTuple_Size_Fast(space, ob)) + "expected %d arguments, got %d", n, PyTuple_GET_SIZE(space, ob)) def wrap_init(space, w_self, w_args, func, w_kwargs): func_init = rffi.cast(initproc, func) Modified: pypy/branch/jitypes2/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/src/getargs.c Thu Dec 9 09:13:06 2010 @@ -453,6 +453,7 @@ strncpy(msgbuf, "is not retrievable", bufsize); return msgbuf; } + PyPy_Borrow(arg, item); msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); /* PySequence_GetItem calls tp->sq_item, which INCREFs */ Modified: pypy/branch/jitypes2/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/state.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/state.py Thu Dec 9 09:13:06 2010 @@ -8,6 +8,7 @@ def __init__(self, space): self.space = space self.reset() + self.programname = lltype.nullptr(rffi.CCHARP.TO) def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef @@ -41,3 +42,16 @@ if always: raise OperationError(self.space.w_SystemError, self.space.wrap( "Function returned an error result without setting an exception")) + + def get_programname(self): + if not self.programname: + space = self.space + argv = space.sys.get('argv') + if space.int_w(space.len(argv)): + argv0 = space.getitem(argv, space.wrap(0)) + progname = space.str_w(argv0) + else: + progname = "pypy" + self.programname = rffi.str2charp(progname) + lltype.render_immortal(self.programname) + return self.programname Modified: pypy/branch/jitypes2/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/stubs.py Thu Dec 9 09:13:06 2010 @@ -668,24 +668,6 @@ """ raise NotImplementedError - at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) -def PyErr_NormalizeException(space, exc, val, tb): - """Under certain circumstances, the values returned by PyErr_Fetch() below - can be "unnormalized", meaning that *exc is a class object but *val is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], PyObject) -def PyErr_SetFromErrnoWithFilename(space, type, filename): - """Similar to PyErr_SetFromErrno(), with the additional behavior that if - filename is not NULL, it is passed to the constructor of type as a third - parameter. In the case of exceptions such as IOError and OSError, - this is used to define the filename attribute of the exception instance. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([rffi.INT_real], PyObject) def PyErr_SetFromWindowsErr(space, ierr): """This is a convenience function to raise WindowsError. If called with @@ -809,21 +791,6 @@ successful invocation of Py_EnterRecursiveCall().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_Check(space, p): - """Return true if its argument is a PyFileObject or a subtype of - PyFileObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_CheckExact(space, p): - """Return true if its argument is a PyFileObject, but not a subtype of - PyFileObject. - """ - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file @@ -857,22 +824,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], PyObject) -def PyFile_GetLine(space, p, n): - """ - - - - Equivalent to p.readline([n]), this function reads one line from the - object p. p may be a file object or any object with a readline() - method. If n is 0, exactly one line is read, regardless of the length of - the line. If n is greater than 0, no more than n bytes will be read - from the file; a partial line can be returned. In both cases, an empty string - is returned if the end of the file is reached immediately. If n is less than - 0, however, one line is read regardless of length, but EOFError is - raised if the end of the file is reached immediately.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyFile_Name(space, p): """Return the name of the file specified by p as a string object.""" @@ -1431,17 +1382,6 @@ raise NotImplementedError @cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) -def Py_GetProgramName(space, ): - """ - - - - Return the program name set with Py_SetProgramName(), or the default. - The returned string points into static storage; the caller should not modify its - value.""" - raise NotImplementedError - - at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPrefix(space, ): """Return the prefix for installed platform-independent files. This is derived through a number of complicated rules from the program name set with Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_cpyext.py Thu Dec 9 09:13:06 2010 @@ -545,17 +545,16 @@ PyObject *true = Py_True; PyObject *tup = NULL; int refcnt = true->ob_refcnt; - int refcnt_middle, refcnt_after; + int refcnt_after; tup = PyTuple_New(1); Py_INCREF(true); if (PyTuple_SetItem(tup, 0, true) < 0) return NULL; - refcnt_middle = true->ob_refcnt; - Py_DECREF(tup); refcnt_after = true->ob_refcnt; - fprintf(stderr, "REFCNT2 %i %i %i\\n", refcnt, refcnt_middle, refcnt_after); - return PyBool_FromLong(refcnt_after == refcnt && refcnt_middle == refcnt+1); + Py_DECREF(tup); + fprintf(stderr, "REFCNT2 %i %i\\n", refcnt, refcnt_after); + return PyBool_FromLong(refcnt_after == refcnt); } static PyMethodDef methods[] = { @@ -683,3 +682,19 @@ assert mod.get_names() == ('cell', 'module', 'property', 'staticmethod', 'builtin_function_or_method') + + def test_get_programname(self): + mod = self.import_extension('foo', [ + ('get_programname', 'METH_NOARGS', + ''' + char* name1 = Py_GetProgramName(); + char* name2 = Py_GetProgramName(); + if (name1 != name2) + Py_RETURN_FALSE; + return PyString_FromString(name1); + ''' + ), + ]) + p = mod.get_programname() + print p + assert 'py' in p Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_intobject.py Thu Dec 9 09:13:06 2010 @@ -19,6 +19,10 @@ assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + assert api.PyInt_AsLong(None) == -1 + assert api.PyErr_Occurred() is space.w_TypeError + api.PyErr_Clear() + assert api.PyInt_AsUnsignedLong(space.wrap(sys.maxint)) == sys.maxint assert api.PyInt_AsUnsignedLong(space.wrap(-5)) == sys.maxint * 2 + 1 assert api.PyErr_Occurred() is space.w_ValueError Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_object.py Thu Dec 9 09:13:06 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ - Py_GE, Py_GT + Py_GE, Py_GT, fopen, fclose, fwrite from pypy.tool.udir import udir class TestObject(BaseApiTest): @@ -188,10 +188,45 @@ rffi.free_charp(filename) rffi.free_charp(mode) + assert api.PyFile_Check(w_file) + assert api.PyFile_CheckExact(w_file) + assert not api.PyFile_Check(space.wrap("text")) + space.call_method(w_file, "write", space.wrap("text")) space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" + def test_file_getline(self, space, api): + filename = rffi.str2charp(str(udir / "_test_file")) + + mode = rffi.str2charp("w") + w_file = api.PyFile_FromString(filename, mode) + space.call_method(w_file, "write", + space.wrap("line1\nline2\nline3\nline4")) + space.call_method(w_file, "close") + + rffi.free_charp(mode) + mode = rffi.str2charp("r") + w_file = api.PyFile_FromString(filename, mode) + rffi.free_charp(filename) + rffi.free_charp(mode) + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "line1\n" + + w_line = api.PyFile_GetLine(w_file, 4) + assert space.str_w(w_line) == "line" + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "2\n" + + # XXX We ought to raise an EOFError here, but don't + w_line = api.PyFile_GetLine(w_file, -1) + # assert api.PyErr_Occurred() is space.w_EOFError + assert space.str_w(w_line) == "line3\n" + + space.call_method(w_file, "close") + class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_pyerrors.py Thu Dec 9 09:13:06 2010 @@ -129,6 +129,41 @@ ]) assert module.check_error() + + def test_normalize(self): + module = self.import_extension('foo', [ + ("check_error", "METH_NOARGS", + ''' + PyObject *type, *val, *tb; + PyErr_SetString(PyExc_TypeError, "message"); + + PyErr_Fetch(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (!PyString_Check(val)) + Py_RETURN_FALSE; + /* Normalize */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + /* Normalize again */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + PyErr_Restore(type, val, tb); + PyErr_Clear(); + Py_RETURN_TRUE; + ''' + ), + ]) + assert module.check_error() + def test_SetFromErrno(self): import sys if sys.platform != 'win32': @@ -149,3 +184,26 @@ except OSError, e: assert e.errno == errno.EBADF assert e.strerror == os.strerror(errno.EBADF) + assert e.filename == None + + def test_SetFromErrnoWithFilename(self): + import sys + if sys.platform != 'win32': + skip("callbacks through ll2ctypes modify errno") + import errno, os + + module = self.import_extension('foo', [ + ("set_from_errno", "METH_NOARGS", + ''' + errno = EBADF; + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "blyf"); + return NULL; + '''), + ], + prologue="#include ") + try: + module.set_from_errno() + except OSError, e: + assert e.filename == "blyf" + assert e.errno == errno.EBADF + assert e.strerror == os.strerror(errno.EBADF) Modified: pypy/branch/jitypes2/pypy/module/cpyext/test/test_tupleobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/test/test_tupleobject.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/test/test_tupleobject.py Thu Dec 9 09:13:06 2010 @@ -7,34 +7,24 @@ class TestTupleObject(BaseApiTest): def test_tupleobject(self, space, api): assert not api.PyTuple_Check(space.w_None) - #assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 XXX + assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 atuple = space.newtuple([0, 1, 'yay']) assert api.PyTuple_Size(atuple) == 3 - #raises(TypeError, api.PyTuple_Size(space.newlist([]))) XXX + assert api.PyTuple_GET_SIZE(atuple) == 3 + raises(TypeError, api.PyTuple_Size(space.newlist([]))) api.PyErr_Clear() def test_tuple_resize(self, space, api): - ref_tup = api.PyTuple_New(3) + py_tuple = api.PyTuple_New(3) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - ar[0] = rffi.cast(PyObject, ref_tup) + ar[0] = rffi.cast(PyObject, make_ref(space, py_tuple)) api._PyTuple_Resize(ar, 2) - assert ar[0] == rffi.cast(PyObject, ref_tup) - # ^^^ our _PyTuple_Resize does not actually need to change the ptr so far - assert api.PyTuple_Size(ar[0]) == 2 + py_tuple = from_ref(space, ar[0]) + assert len(py_tuple.wrappeditems) == 2 api._PyTuple_Resize(ar, 10) - assert api.PyTuple_Size(ar[0]) == 10 + py_tuple = from_ref(space, ar[0]) + assert len(py_tuple.wrappeditems) == 10 api.Py_DecRef(ar[0]) lltype.free(ar, flavor='raw') - - def test_tuple_setup(self, space, api): - ref_tup = api.PyTuple_New(2) - ref0 = make_ref(space, space.wrap(123)) - api.PyTuple_SetItem(ref_tup, 0, ref0) - ref1 = make_ref(space, space.wrap(456)) - api.PyTuple_SetItem(ref_tup, 1, ref1) - - w_tup = from_ref(space, ref_tup) - assert space.is_true(space.eq(w_tup, space.wrap((123, 456)))) - api.Py_DecRef(ref_tup) Modified: pypy/branch/jitypes2/pypy/module/cpyext/tupleobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/cpyext/tupleobject.py (original) +++ pypy/branch/jitypes2/pypy/module/cpyext/tupleobject.py Thu Dec 9 09:13:06 2010 @@ -1,144 +1,55 @@ from pypy.interpreter.error import OperationError from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers, PyObjectFields, - cpython_struct, bootstrap_function) + build_type_checkers) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - borrow_from, make_ref, from_ref, make_typedescr, get_typedescr, Reference, - track_reference) + borrow_from, make_ref, from_ref) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall +from pypy.objspace.std.tupleobject import W_TupleObject -## -## Implementation of PyTupleObject -## =============================== -## -## We have the same problem as PyStringObject: a PyTupleObject can be -## initially used in a read-write way with PyTuple_New(), PyTuple_SetItem() -## and _PyTuple_Resize(). -## -## The 'size' and 'items' fields of a PyTupleObject are always valid. -## Apart from that detail, see the big comment in stringobject.py for -## more information. -## - -ARRAY_OF_PYOBJ = rffi.CArrayPtr(PyObject) -PyTupleObjectStruct = lltype.ForwardReference() -PyTupleObject = lltype.Ptr(PyTupleObjectStruct) -PyTupleObjectFields = PyObjectFields + \ - (("items", ARRAY_OF_PYOBJ), ("size", Py_ssize_t)) -cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) - - at bootstrap_function -def init_tupleobject(space): - "Type description of PyTupleObject" - make_typedescr(space.w_tuple.instancetypedef, - basestruct=PyTupleObject.TO, - attach=tuple_attach, - dealloc=tuple_dealloc, - realize=tuple_realize) PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple") -def new_empty_tuple(space, length): - """ - Allocate a PyTupleObject and its array, but without a corresponding - interpreter object. The array items may be mutated, until - tuple_realize() is called. - """ - typedescr = get_typedescr(space.w_tuple.instancetypedef) - py_obj = typedescr.allocate(space, space.w_tuple) - py_tup = rffi.cast(PyTupleObject, py_obj) - - py_tup.c_items = lltype.malloc(ARRAY_OF_PYOBJ.TO, length, - flavor='raw', zero=True) - py_tup.c_size = length - return py_tup - -def tuple_attach(space, py_obj, w_obj): - """ - Fills a newly allocated PyTupleObject with the given tuple object. - """ - items_w = space.fixedview(w_obj) - py_tup = rffi.cast(PyTupleObject, py_obj) - py_tup.c_items = lltype.nullptr(ARRAY_OF_PYOBJ.TO) - py_tup.c_size = len(items_w) - -def tuple_realize(space, py_obj): - """ - Creates the tuple in the interpreter. The PyTupleObject items array - must not be modified after this call. - """ - py_tup = rffi.cast(PyTupleObject, py_obj) - # If your CPython extension creates a self-referential tuple - # with PyTuple_SetItem(), you loose. - c_items = py_tup.c_items - items_w = [from_ref(space, c_items[i]) for i in range(py_tup.c_size)] - w_obj = space.newtuple(items_w) - track_reference(space, py_obj, w_obj) - return w_obj - - at cpython_api([PyObject], lltype.Void, external=False) -def tuple_dealloc(space, py_obj): - """Frees allocated PyTupleObject resources. - """ - py_tup = rffi.cast(PyTupleObject, py_obj) - if py_tup.c_items: - for i in range(py_tup.c_size): - Py_DecRef(space, py_tup.c_items[i]) - lltype.free(py_tup.c_items, flavor="raw") - from pypy.module.cpyext.object import PyObject_dealloc - PyObject_dealloc(space, py_obj) - -#_______________________________________________________________________ - @cpython_api([Py_ssize_t], PyObject) def PyTuple_New(space, size): - return rffi.cast(PyObject, new_empty_tuple(space, size)) + return space.newtuple([space.w_None] * size) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PyTuple_SetItem(space, ref, pos, ref_item): - # XXX steals a reference at the level of PyObjects. Don't try to - # XXX call this function with an interpreter object as ref_item! - - # XXX do PyTuple_Check, without forcing ref as an interpreter object - # XXX -- then if it fails it should also steal a reference, test it!!! - ref_tup = rffi.cast(PyTupleObject, ref) - if not ref_tup.c_items: - msg = "PyTuple_SetItem() called on an already-escaped tuple object" - raise OperationError(space.w_SystemError, space.wrap(msg)) - ref_old = ref_tup.c_items[pos] - ref_tup.c_items[pos] = ref_item # SetItem steals a reference! - Py_DecRef(space, ref_old) +def PyTuple_SetItem(space, w_t, pos, w_obj): + if not PyTuple_Check(space, w_t): + # XXX this should also steal a reference, test it!!! + PyErr_BadInternalCall(space) + assert isinstance(w_t, W_TupleObject) + w_t.wrappeditems[pos] = w_obj + Py_DecRef(space, w_obj) # SetItem steals a reference! return 0 @cpython_api([PyObject, Py_ssize_t], PyObject) -def PyTuple_GetItem(space, ref, pos): - # XXX do PyTuple_Check, without forcing ref as an interpreter object - ref_tup = rffi.cast(PyTupleObject, ref) - if ref_tup.c_items: - return Reference(ref_tup.c_items[pos]) # borrowed reference - else: - w_t = from_ref(space, ref) - w_obj = space.getitem(w_t, space.wrap(pos)) - return borrow_from(w_t, w_obj) - - at cpython_api([PyObject], Py_ssize_t, error=-1) -def _PyTuple_Size_Fast(space, ref): - # custom version: it's not a macro, so it can be called from other .py - # files; but it doesn't include PyTuple_Check() - ref_tup = rffi.cast(PyTupleObject, ref) - return ref_tup.c_size +def PyTuple_GetItem(space, w_t, pos): + if not PyTuple_Check(space, w_t): + PyErr_BadInternalCall(space) + assert isinstance(w_t, W_TupleObject) + w_obj = w_t.wrappeditems[pos] + return borrow_from(w_t, w_obj) + + at cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) +def PyTuple_GET_SIZE(space, w_t): + """Return the size of the tuple p, which must be non-NULL and point to a tuple; + no error checking is performed. """ + assert isinstance(w_t, W_TupleObject) + return len(w_t.wrappeditems) @cpython_api([PyObject], Py_ssize_t, error=-1) def PyTuple_Size(space, ref): """Take a pointer to a tuple object, and return the size of that tuple.""" - # XXX do PyTuple_Check, without forcing ref as an interpreter object - ref_tup = rffi.cast(PyTupleObject, ref) - return ref_tup.c_size + if not PyTuple_Check(space, ref): + raise OperationError(space.w_TypeError, + space.wrap("expected tuple object")) + return PyTuple_GET_SIZE(space, ref) @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyTuple_Resize(space, refp, newsize): +def _PyTuple_Resize(space, ref, newsize): """Can be used to resize a tuple. newsize will be the new length of the tuple. Because tuples are supposed to be immutable, this should only be used if there is only one reference to the object. Do not use this if the tuple may already @@ -149,22 +60,18 @@ this function. If the object referenced by *p is replaced, the original *p is destroyed. On failure, returns -1 and sets *p to NULL, and raises MemoryError or SystemError.""" - # XXX do PyTuple_Check, without forcing ref as an interpreter object - # XXX -- then if it fails it should reset refp[0] to null - ref_tup = rffi.cast(PyTupleObject, refp[0]) - c_newitems = lltype.malloc(ARRAY_OF_PYOBJ.TO, newsize, - flavor='raw', zero=True) - c_olditems = ref_tup.c_items - if not c_olditems: - msg = "_PyTuple_Resize() called on an already-escaped tuple object" - raise OperationError(space.w_SystemError, space.wrap(msg)) - oldsize = ref_tup.c_size - for i in range(min(oldsize, newsize)): - c_newitems[i] = c_olditems[i] - # decref items deleted by shrinkage - for i in range(newsize, oldsize): - Py_DecRef(space, c_olditems[i]) - ref_tup.c_items = c_newitems - ref_tup.c_size = newsize - lltype.free(c_olditems, flavor='raw') + py_tuple = from_ref(space, ref[0]) + if not PyTuple_Check(space, py_tuple): + PyErr_BadInternalCall(space) + assert isinstance(py_tuple, W_TupleObject) + py_newtuple = PyTuple_New(space, newsize) + + to_cp = newsize + oldsize = len(py_tuple.wrappeditems) + if oldsize < newsize: + to_cp = oldsize + for i in range(to_cp): + py_newtuple.wrappeditems[i] = py_tuple.wrappeditems[i] + Py_DecRef(space, ref[0]) + ref[0] = make_ref(space, py_newtuple) return 0 Modified: pypy/branch/jitypes2/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/jitypes2/pypy/module/exceptions/interp_exceptions.py Thu Dec 9 09:13:06 2010 @@ -452,6 +452,7 @@ self.w_text = space.w_None self.w_msg = space.wrap('') self.w_print_file_and_line = space.w_None # what's that? + self.w_lastlineno = space.w_None # this is a pypy extension W_BaseException.__init__(self, space) def descr_init(self, space, args_w): @@ -459,11 +460,12 @@ if len(args_w) > 0: self.w_msg = args_w[0] if len(args_w) == 2: - values_w = space.fixedview(args_w[1], 4) - self.w_filename = values_w[0] - self.w_lineno = values_w[1] - self.w_offset = values_w[2] - self.w_text = values_w[3] + values_w = space.fixedview(args_w[1]) + if len(values_w) > 0: self.w_filename = values_w[0] + if len(values_w) > 1: self.w_lineno = values_w[1] + if len(values_w) > 2: self.w_offset = values_w[2] + if len(values_w) > 3: self.w_text = values_w[3] + if len(values_w) > 4: self.w_lastlineno = values_w[4] W_BaseException.descr_init(self, space, args_w) descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] @@ -472,18 +474,24 @@ if type(self.msg) is not str: return str(self.msg) + lineno = None buffer = self.msg have_filename = type(self.filename) is str - have_lineno = type(self.lineno) is int + if type(self.lineno) is int: + if (type(self.lastlineno) is int and + self.lastlineno > self.lineno): + lineno = 'lines %d-%d' % (self.lineno, self.lastlineno) + else: + lineno = 'line %d' % (self.lineno,) if have_filename: import os fname = os.path.basename(self.filename or "???") - if have_lineno: - buffer = "%s (%s, line %ld)" % (self.msg, fname, self.lineno) + if lineno: + buffer = "%s (%s, %s)" % (self.msg, fname, lineno) else: buffer ="%s (%s)" % (self.msg, fname) - elif have_lineno: - buffer = "%s (line %ld)" % (self.msg, self.lineno) + elif lineno: + buffer = "%s (%s)" % (self.msg, lineno) return buffer """) @@ -504,6 +512,7 @@ text = readwrite_attrproperty_w('w_text', W_SyntaxError), print_file_and_line = readwrite_attrproperty_w('w_print_file_and_line', W_SyntaxError), + lastlineno = readwrite_attrproperty_w('w_lastlineno', W_SyntaxError), ) W_FutureWarning = _new_exception('FutureWarning', W_Warning, Modified: pypy/branch/jitypes2/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/imp/importing.py (original) +++ pypy/branch/jitypes2/pypy/module/imp/importing.py Thu Dec 9 09:13:06 2010 @@ -529,7 +529,7 @@ space.sys.setmodule(w_module) raise finally: - space.reloading_modules.clear() + del space.reloading_modules[modulename] # __________________________________________________________________ Modified: pypy/branch/jitypes2/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/jitypes2/pypy/module/pypyjit/test/test_pypy_c.py Thu Dec 9 09:13:06 2010 @@ -909,11 +909,14 @@ def test_array_sum(self): for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): res = 19352859 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array @@ -961,11 +964,14 @@ print '='*65 print '='*20, 'running test for tc=%r' % (tc,), '='*20 res = 73574560 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array Modified: pypy/branch/jitypes2/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/jitypes2/pypy/module/sys/vm.py (original) +++ pypy/branch/jitypes2/pypy/module/sys/vm.py Thu Dec 9 09:13:06 2010 @@ -41,27 +41,21 @@ f = ec.getnextframe_nohidden(f) return space.wrap(f) -# directly from the C code in ceval.c, might be moved somewhere else. - def setrecursionlimit(space, w_new_limit): - """Set the maximum depth of the Python interpreter stack to n. This -limit prevents infinite recursion from causing an overflow of the C -stack and crashing Python. The highest possible limit is platform -dependent.""" + """DEPRECATED on PyPy. Will issue warning and not work + """ new_limit = space.int_w(w_new_limit) if new_limit <= 0: raise OperationError(space.w_ValueError, space.wrap("recursion limit must be positive")) # global recursion_limit # we need to do it without writing globals. + space.warn('setrecursionlimit deprecated', space.w_DeprecationWarning) space.sys.recursionlimit = new_limit def getrecursionlimit(space): - """Return the current value of the recursion limit, the maximum depth - of the Python interpreter stack. This limit prevents infinite - recursion from causing an overflow of the C stack and crashing Python. + """DEPRECATED on PyPy. Will issue warning and not work """ - return space.wrap(space.sys.recursionlimit) def setcheckinterval(space, interval): Modified: pypy/branch/jitypes2/pypy/objspace/std/complexobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/complexobject.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/complexobject.py Thu Dec 9 09:13:06 2010 @@ -23,90 +23,92 @@ """ representation for debugging purposes """ return "" % (w_self.realval, w_self.imagval) -registerimplementation(W_ComplexObject) - -c_1 = (1.0, 0.0) + def sub(self, other): + return W_ComplexObject(self.realval - other.realval, + self.imagval - other.imagval) + + def mul(self, other): + r = self.realval * other.realval - self.imagval * other.imagval + i = self.realval * other.imagval + self.imagval * other.realval + return W_ComplexObject(r, i) + + def div(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 < 0: + abs_r2 = - r2 + else: + abs_r2 = r2 + if i2 < 0: + abs_i2 = - i2 + else: + abs_i2 = i2 + if abs_r2 >= abs_i2: + if abs_r2 == 0.0: + raise ZeroDivisionError + else: + ratio = i2 / r2 + denom = r2 + i2 * ratio + rr = (r1 + i1 * ratio) / denom + ir = (i1 - r1 * ratio) / denom + else: + ratio = r2 / i2 + denom = r2 * ratio + i2 + assert i2 != 0.0 + rr = (r1 * ratio + i1) / denom + ir = (i1 * ratio - r1) / denom + return W_ComplexObject(rr,ir) + + def divmod(self, other): + w_div = self.div(other) + div = math.floor(w_div.realval) + w_mod = self.sub( + W_ComplexObject(other.realval * div, other.imagval * div)) + return (W_ComplexObject(div, 0), w_mod) + + def pow(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 == 0.0 and i2 == 0.0: + rr, ir = 1, 0 + elif r1 == 0.0 and i1 == 0.0: + if i2 != 0.0 or r2 < 0.0: + raise ZeroDivisionError + rr, ir = (0.0, 0.0) + else: + vabs = math.hypot(r1,i1) + len = math.pow(vabs,r2) + at = math.atan2(i1,r1) + phase = at * r2 + if i2 != 0.0: + len /= math.exp(at * i2) + phase += i2 * math.log(vabs) + rr = len * math.cos(phase) + ir = len * math.sin(phase) + return W_ComplexObject(rr, ir) + + def pow_int(self, n): + if n > 100 or n < -100: + return self.pow(W_ComplexObject(1.0 * n, 0.0)) + elif n > 0: + return self.pow_positive_int(n) + else: + return w_one.div(self.pow_positive_int(-n)) -def _sum(c1, c2): - return (c1[0]+c2[0],c1[1]+c2[1]) + def pow_positive_int(self, n): + mask = 1 + w_result = w_one + while mask > 0 and n >= mask: + if n & mask: + w_result = w_result.mul(self) + mask <<= 1 + self = self.mul(self) -def _diff(c1, c2): - return (c1[0]-c2[0],c1[1]-c2[1]) + return w_result -def _prod(c1, c2): - r = c1[0]*c2[0] - c1[1]*c2[1] - i = c1[0]*c2[1] + c1[1]*c2[0] - return (r,i) - -def _quot(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 < 0: - abs_r2 = - r2 - else: - abs_r2 = r2 - if i2 < 0: - abs_i2 = - i2 - else: - abs_i2 = i2 - if abs_r2 >= abs_i2: - if abs_r2 == 0.0: - raise ZeroDivisionError - else: - ratio = i2 / r2 - denom = r2 + i2 * ratio - rr = (r1 + i1 * ratio) / denom - ir = (i1 - r1 * ratio) / denom - else: - ratio = r2 / i2 - denom = r2 * ratio + i2 - assert i2 != 0.0 - rr = (r1 * ratio + i1) / denom - ir = (i1 * ratio - r1) / denom - return (rr,ir) - -def _pow(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 == 0.0 and i2 == 0.0: - rr, ir = c_1 - elif r1 == 0.0 and i1 == 0.0: - if i2 != 0.0 or r2 < 0.0: - raise ZeroDivisionError - rr, ir = (0.0, 0.0) - else: - vabs = math.hypot(r1,i1) - len = math.pow(vabs,r2) - at = math.atan2(i1,r1) - phase = at * r2 - if i2 != 0.0: - len /= math.exp(at * i2) - phase += i2 * math.log(vabs) - rr = len * math.cos(phase) - ir = len * math.sin(phase) - return (rr, ir) - -def _powu(c,n): - mask = 1; - rr, ir = c_1 - rp = c[0] - ip = c[1] - while mask > 0 and n >= mask: - if n & mask: - rr, ir = _prod((rr, ir), (rp, ip)) - mask <<= 1 - rp, ip = _prod((rp, ip), (rp, ip)) - - return (rr, ir) - -def _powi(c,n): - if n > 100 or n < -100: - return _pow(c,(1.0 * n, 0.0)) - elif n > 0: - return _powu(c, n) - else: - return _quot(c_1, _powu(c, -n)) +registerimplementation(W_ComplexObject) +w_one = W_ComplexObject(1, 0) def delegate_Bool2Complex(space, w_bool): @@ -126,38 +128,25 @@ return W_ComplexObject(w_float.floatval, 0.0) def hash__Complex(space, w_value): - #this is straight out of CPython complex implementation - hashreal = _hash_float(space, w_value.realval) - if hashreal == -1: - return space.newint(-1) hashimg = _hash_float(space, w_value.imagval) - if hashimg == -1: - return space.newint(-1) combined = hashreal + 1000003 * hashimg - if (combined == -1): - combined = -2 return space.newint(combined) -def _w2t(space, w_complex): - "convert an interplevel complex object to a tuple representation" - return w_complex.realval, w_complex.imagval - -def _t2w(space, c): - return W_ComplexObject(c[0], c[1]) - def add__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _sum(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval + w_complex2.realval, + w_complex1.imagval + w_complex2.imagval) def sub__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _diff(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval - w_complex2.realval, + w_complex1.imagval - w_complex2.imagval) def mul__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _prod(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.mul(w_complex2) def div__Complex_Complex(space, w_complex1, w_complex2): try: - return _t2w(space, _quot(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.div(w_complex2) except ZeroDivisionError, e: raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) @@ -165,49 +154,38 @@ def mod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[1] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex remainder")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - - return _t2w(space, mod) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) def divmod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + div, mod = w_complex1.divmod(w_complex2) except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex divmod()")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - w_div = _t2w(space, div) - w_mod = _t2w(space, mod) - return space.newtuple([w_div, w_mod]) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) + return space.newtuple([div, mod]) def floordiv__Complex_Complex(space, w_complex1, w_complex2): + # don't care about the slight slowdown you get from using divmod try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[0] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex floordiv()")) - div = (math.floor(div[0]), 0.0) - return _t2w(space, div) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) -def pow__Complex_Complex_ANY(space, w_complex1, w_complex2, thirdArg): +def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) + int_exponent = int(w_exponent.realval) try: - v = _w2t(space, w_complex1) - exponent = _w2t(space, w_complex2) - int_exponent = int(exponent[0]) - if exponent[1] == 0.0 and exponent[0] == int_exponent: - p = _powi(v, int_exponent) + if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: + w_p = w_complex.pow_int(int_exponent) else: - p = _pow(v, exponent) + w_p = w_complex.pow(w_exponent) except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("0.0 to a negative or complex power")) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("complex exponentiation")) - return _t2w(space, p) + return w_p def neg__Complex(space, w_complex): return W_ComplexObject(-w_complex.realval, -w_complex.imagval) Modified: pypy/branch/jitypes2/pypy/objspace/std/complextype.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/complextype.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/complextype.py Thu Dec 9 09:13:06 2010 @@ -1,7 +1,7 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.strutil import interp_string_to_float, ParseStringError +from pypy.objspace.std.strutil import string_to_float, ParseStringError from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod @@ -131,8 +131,8 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) try: - realval = interp_string_to_float(space, realstr) - imagval = interp_string_to_float(space, imagstr) + realval = string_to_float(realstr) + imagval = string_to_float(imagstr) except ParseStringError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) else: Modified: pypy/branch/jitypes2/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/floattype.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/floattype.py Thu Dec 9 09:13:06 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.strutil import ParseStringError -from pypy.objspace.std.strutil import interp_string_to_float +from pypy.objspace.std.strutil import string_to_float def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject @@ -10,7 +10,7 @@ if space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) @@ -21,7 +21,7 @@ from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) Modified: pypy/branch/jitypes2/pypy/objspace/std/longobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/longobject.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/longobject.py Thu Dec 9 09:13:06 2010 @@ -45,19 +45,6 @@ fromrarith_int._annspecialcase_ = "specialize:argtype(0)" fromrarith_int = staticmethod(fromrarith_int) - def fromdecimalstr(s): - return W_LongObject(rbigint.fromdecimalstr(s)) - fromdecimalstr = staticmethod(fromdecimalstr) - - def _count_bits(self): - return self.num._count_bits() - - def is_odd(self): - return self.num.is_odd() - - def get_sign(self): - return self.num.sign - registerimplementation(W_LongObject) # bool-to-long Modified: pypy/branch/jitypes2/pypy/objspace/std/strutil.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/strutil.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/strutil.py Thu Dec 9 09:13:06 2010 @@ -150,7 +150,7 @@ del calc_mantissa_bits MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1 -def interp_string_to_float(space, s): +def string_to_float(s): """ Conversion of string to float. This version tries to only raise on invalid literals. @@ -162,10 +162,9 @@ s = strip_spaces(s) if not s: - raise OperationError(space.w_ValueError, space.wrap( - "empty string for float()")) + raise ParseStringError("empty string for float()") + - low = s.lower() if low == "-inf": return -INFINITY @@ -204,68 +203,56 @@ if len(digits) == 0: digits = '0' - # a few abbreviations - from pypy.objspace.std import longobject - mklong = longobject.W_LongObject.fromint - d2long = longobject.W_LongObject.fromdecimalstr - adlong = longobject.add__Long_Long - longup = longobject.pow__Long_Long_None - multip = longobject.mul__Long_Long - divide = longobject.div__Long_Long - lshift = longobject.lshift__Long_Long - rshift = longobject.rshift__Long_Long - # 4) compute the exponent and truncate to +-400 if not exponent: exponent = '0' - w_le = d2long(exponent) - w_le = adlong(space, w_le, mklong(space, dexp)) + long_exponent = rbigint.fromdecimalstr(exponent) + long_exponent = long_exponent.add(rbigint.fromint(dexp)) try: - e = w_le.toint() + e = long_exponent.toint() except OverflowError: # XXX poking at internals - e = w_le.num.sign * 400 - if e >= 400: - e = 400 - elif e <= -400: - e = -400 + e = long_exponent.sign * 400 + else: + if e >= 400: + e = 400 + elif e <= -400: + e = -400 # 5) compute the value using long math and proper rounding. - w_lr = d2long(digits) - w_10 = mklong(space, 10) - w_1 = mklong(space, 1) + b_digits = rbigint.fromdecimalstr(digits) + b_10 = rbigint.fromint(10) + b_1 = rbigint.fromint(1) if e >= 0: bits = 0 - w_pten = longup(space, w_10, mklong(space, e), space.w_None) - w_m = multip(space, w_lr, w_pten) + b_power_of_ten = b_10.pow(rbigint.fromint(e)) + b_mantissa = b_digits.mul(b_power_of_ten) else: # compute a sufficiently large scale prec = MANTISSA_DIGITS * 2 + 22 # 128, maybe bits = - (int(math.ceil(-e / math.log10(2.0) - 1e-10)) + prec) - w_scale = lshift(space, w_1, mklong(space, -bits)) - w_pten = longup(space, w_10, mklong(space, -e), None) - w_tmp = multip(space, w_lr, w_scale) - w_m = divide(space, w_tmp, w_pten) + b_scale = b_1.lshift(-bits) + b_power_of_ten = b_10.pow(rbigint.fromint(-e)) + b_mantissa = b_digits.mul(b_scale).div(b_power_of_ten) # we now have a fairly large mantissa. # Shift it and round the last bit. # first estimate the bits and do a big shift - mbits = w_m._count_bits() + mbits = b_mantissa._count_bits() needed = MANTISSA_BITS if mbits > needed: if mbits > needed+1: shifted = mbits - (needed+1) - w_m = rshift(space, w_m, mklong(space, shifted)) + b_mantissa = b_mantissa.rshift(shifted) bits += shifted # do the rounding bits += 1 - round = w_m.is_odd() - w_m = rshift(space, w_m, w_1) - w_m = adlong(space, w_m, mklong(space, round)) + round = b_mantissa.is_odd() + b_mantissa = b_mantissa.rshift(1).add(rbigint.fromint(round)) try: - r = math.ldexp(w_m.tofloat(), bits) + r = math.ldexp(b_mantissa.tofloat(), bits) # XXX I guess we do not check for overflow in ldexp as we agreed to! if r == 2*r and r != 0.0: raise OverflowError Modified: pypy/branch/jitypes2/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/test/test_complexobject.py Thu Dec 9 09:13:06 2010 @@ -1,5 +1,6 @@ import py -from pypy.objspace.std import complexobject as cobj +from pypy.objspace.std.complexobject import W_ComplexObject, \ + pow__Complex_Complex_ANY from pypy.objspace.std import complextype as cobjtype from pypy.objspace.std.multimethod import FailedToImplement from pypy.objspace.std.stringobject import W_StringObject @@ -11,7 +12,7 @@ def _test_instantiation(self): def _t_complex(r=0.0,i=0.0): - c = cobj.W_ComplexObject(r, i) + c = W_ComplexObject(r, i) assert c.real == float(r) and c.imag == float(i) pairs = ( (1, 1), @@ -38,21 +39,31 @@ test_cparse('.e+5', '.e+5', '0.0') def test_pow(self): - assert cobj._pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) - assert cobj._pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) - rr, ir = cobj._pow((0.0,1.0),(2.0,0.0)) + def _pow((r1, i1), (r2, i2)): + w_res = W_ComplexObject(r1, i1).pow(W_ComplexObject(r2, i2)) + return w_res.realval, w_res.imagval + assert _pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) + assert _pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) + rr, ir = _pow((0.0,1.0),(2.0,0.0)) assert abs(-1.0 - rr) < EPS assert abs(0.0 - ir) < EPS - assert cobj._powu((0.0,2.0),0) == (1.0,0.0) - assert cobj._powu((0.0,0.0),2) == (0.0,0.0) - assert cobj._powu((0.0,1.0),2) == (-1.0,0.0) - assert cobj._powi((0.0,2.0),0) == (1.0,0.0) - assert cobj._powi((0.0,0.0),2) == (0.0,0.0) - assert cobj._powi((0.0,1.0),2) == (-1.0,0.0) - c = cobj.W_ComplexObject(0.0,1.0) - p = cobj.W_ComplexObject(2.0,0.0) - r = cobj.pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) + def _powu((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_positive_int(n) + return w_res.realval, w_res.imagval + assert _powu((0.0,2.0),0) == (1.0,0.0) + assert _powu((0.0,0.0),2) == (0.0,0.0) + assert _powu((0.0,1.0),2) == (-1.0,0.0) + + def _powi((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_int(n) + return w_res.realval, w_res.imagval + assert _powi((0.0,2.0),0) == (1.0,0.0) + assert _powi((0.0,0.0),2) == (0.0,0.0) + assert _powi((0.0,1.0),2) == (-1.0,0.0) + c = W_ComplexObject(0.0,1.0) + p = W_ComplexObject(2.0,0.0) + r = pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) assert r.realval == -1.0 assert r.imagval == 0.0 Modified: pypy/branch/jitypes2/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/branch/jitypes2/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/branch/jitypes2/pypy/objspace/std/test/test_strutil.py Thu Dec 9 09:13:06 2010 @@ -131,8 +131,6 @@ assert string_to_bigint('1891234174197319').tolong() == 1891234174197319 def test_string_to_float(self): - def string_to_float(x): - return interp_string_to_float(self.space, x) assert string_to_float('0') == 0.0 assert string_to_float('1') == 1.0 assert string_to_float('-1.5') == -1.5 @@ -180,3 +178,4 @@ print repr(s) if s.strip(): # empty s raises OperationError directly py.test.raises(ParseStringError, string_to_float, s) + py.test.raises(ParseStringError, string_to_float, "") Modified: pypy/branch/jitypes2/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/jitypes2/pypy/rpython/lltypesystem/ll2ctypes.py Thu Dec 9 09:13:06 2010 @@ -409,6 +409,7 @@ subcls = get_common_subclass(mixin_cls, instance.__class__) instance.__class__ = subcls instance._storage = ctypes_storage + assert ctypes_storage # null pointer? class _parentable_mixin(object): """Mixin added to _parentable containers when they become ctypes-based. Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/base.py Thu Dec 9 09:13:06 2010 @@ -36,6 +36,13 @@ self.finalizer_lock_count = 0 self.run_finalizers = self.AddressDeque() + def post_setup(self): + # More stuff that needs to be initialized when the GC is already + # fully working. (Only called by gctransform/framework for now.) + from pypy.rpython.memory.gc import env + if env.read_from_env('PYPY_GC_DEBUG') > 0: + self.DEBUG = True + def _teardown(self): pass @@ -48,7 +55,8 @@ # The following flag enables costly consistency checks after each # collection. It is automatically set to True by test_gc.py. The # checking logic is translatable, so the flag can be set to True - # here before translation. + # here before translation. At run-time, if PYPY_GC_DEBUG is set, + # then it is also set to True. DEBUG = False def set_query_functions(self, is_varsize, has_gcptr_in_varsize, Modified: pypy/branch/jitypes2/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gc/minimark.py Thu Dec 9 09:13:06 2010 @@ -32,6 +32,9 @@ limit. Useful to avoid spending all the time in the GC in very small programs. Defaults to 8 times the nursery. + + PYPY_GC_DEBUG Enable extra checks around collections that are + too slow for normal use. """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -841,7 +844,7 @@ def remember_young_pointer(addr_struct, newvalue): # 'addr_struct' is the address of the object in which we write. # 'newvalue' is the address that we are going to write in there. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") # @@ -878,7 +881,7 @@ # 'addr_array' is the address of the object in which we write, # which must have an array part; 'index' is the index of the # item that is (or contains) the pointer that we write. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_array), "nursery array with GCFLAG_NO_YOUNG_PTRS") objhdr = self.header(addr_array) Modified: pypy/branch/jitypes2/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/jitypes2/pypy/rpython/memory/gctransform/framework.py Thu Dec 9 09:13:06 2010 @@ -189,6 +189,7 @@ # run-time initialization code root_walker.setup_root_walker() gcdata.gc.setup() + gcdata.gc.post_setup() def frameworkgc__teardown(): # run-time teardown code for tests! Modified: pypy/branch/jitypes2/pypy/rpython/tool/rffi_platform.py ============================================================================== --- pypy/branch/jitypes2/pypy/rpython/tool/rffi_platform.py (original) +++ pypy/branch/jitypes2/pypy/rpython/tool/rffi_platform.py Thu Dec 9 09:13:06 2010 @@ -639,7 +639,6 @@ C_HEADER = """ #include #include /* for offsetof() */ -#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); Modified: pypy/branch/jitypes2/pypy/tool/error.py ============================================================================== --- pypy/branch/jitypes2/pypy/tool/error.py (original) +++ pypy/branch/jitypes2/pypy/tool/error.py Thu Dec 9 09:13:06 2010 @@ -120,7 +120,7 @@ msg.append(" (%s getting at the binding!)" % ( e.__class__.__name__,)) return - for desc in descs.keys(): + for desc in list(descs): func = desc.pyobj if func is None: r = repr(desc) Modified: pypy/branch/jitypes2/pypy/tool/release/package.py ============================================================================== --- pypy/branch/jitypes2/pypy/tool/release/package.py (original) +++ pypy/branch/jitypes2/pypy/tool/release/package.py Thu Dec 9 09:13:06 2010 @@ -78,7 +78,7 @@ old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) # ignore errors + os.system("strip -x " + str(archive_pypy_c)) # ignore errors if USE_TARFILE_MODULE: import tarfile tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') Modified: pypy/branch/jitypes2/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/goal/app_main.py (original) +++ pypy/branch/jitypes2/pypy/translator/goal/app_main.py Thu Dec 9 09:13:06 2010 @@ -312,8 +312,9 @@ cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 - # but we need more in the translated PyPy for the compiler package - sys.setrecursionlimit(5000) + # but we need more in the translated PyPy for the compiler package + if '__pypy__' not in sys.builtin_module_names: + sys.setrecursionlimit(5000) if unbuffered: set_unbuffered_io() Modified: pypy/branch/jitypes2/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/jitypes2/pypy/translator/platform/darwin.py (original) +++ pypy/branch/jitypes2/pypy/translator/platform/darwin.py Thu Dec 9 09:13:06 2010 @@ -11,8 +11,10 @@ shared_only = () so_ext = 'so' - - default_cc = 'gcc' + + # NOTE: GCC 4.2 will fail at runtime due to subtle issues, possibly + # related to GC roots. Using LLVM-GCC or Clang will break the build. + default_cc = 'gcc-4.0' def __init__(self, cc=None): if cc is None: @@ -87,4 +89,3 @@ link_flags = ('-arch', 'x86_64', '-mmacosx-version-min=10.4') cflags = ('-arch', 'x86_64', '-O3', '-fomit-frame-pointer', '-mmacosx-version-min=10.4') - default_cc = 'gcc-4.0' From arigo at codespeak.net Thu Dec 9 11:43:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 11:43:21 +0100 (CET) Subject: [pypy-svn] r79916 - pypy/branch/gc-debug/pypy/rpython/memory/gc Message-ID: <20101209104321.45F9B282BDC@codespeak.net> Author: arigo Date: Thu Dec 9 11:43:19 2010 New Revision: 79916 Modified: pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py Log: Allow PYPY_GC_NURSERY to be set to any value, like 100 or 10K. If it is less than the minimum nursery size it will simply bump forward the current nursery pointer after every minor collection, so that only the specified number of bytes is left. Modified: pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/gc-debug/pypy/rpython/memory/gc/minimark.py Thu Dec 9 11:43:19 2010 @@ -3,9 +3,8 @@ Environment variables can be used to fine-tune the following parameters: PYPY_GC_NURSERY The nursery size. Defaults to half the size of - the L2 cache. Try values like '1.2MB'. Minimum - is about 64KB. Use 1 to force every malloc to - do a minor collection for debugging. + the L2 cache. Try values like '1.2MB'. Small values + (like 1 or 1KB) are useful for debugging. PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', which means trigger a major collection when the @@ -230,7 +229,7 @@ self.nursery = NULL self.nursery_free = NULL self.nursery_top = NULL - self.debug_always_do_minor_collect = False + self.debug_tiny_nursery = -1 self.debug_rotating_nurseries = None # # The ArenaCollection() handles the nonmovable objects allocation. @@ -299,15 +298,20 @@ # From there on, the GC is fully initialized and the code # below can use it newsize = env.read_from_env('PYPY_GC_NURSERY') - # PYPY_GC_NURSERY=1 forces a minor collect for every malloc. - # Useful to debug external factors, like trackgcroot or the - # handling of the write barrier. - self.debug_always_do_minor_collect = newsize == 1 + # PYPY_GC_NURSERY=smallvalue means that minor collects occur + # very frequently; the extreme case is PYPY_GC_NURSERY=1, which + # forces a minor collect for every malloc. Useful to debug + # external factors, like trackgcroot or the handling of the write + # barrier. Implemented by still using 'minsize' for the nursery + # size (needed to handle e.g. mallocs of 8249 words) but hacking + # at the current nursery position in collect_and_reserve(). if newsize <= 0: newsize = env.estimate_best_nursery_size() if newsize <= 0: newsize = defaultsize - newsize = max(newsize, minsize) + if newsize < minsize: + self.debug_tiny_nursery = newsize & ~(WORD-1) + newsize = minsize # major_coll = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT') if major_coll > 1.0: @@ -561,8 +565,9 @@ self.nursery_free = result + totalsize ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow") # - if self.debug_always_do_minor_collect: - self.nursery_free = self.nursery_top + if self.debug_tiny_nursery >= 0: # for debugging + if self.nursery_top - self.nursery_free > self.debug_tiny_nursery: + self.nursery_free = self.nursery_top - self.debug_tiny_nursery # return result collect_and_reserve._dont_inline_ = True From arigo at codespeak.net Thu Dec 9 13:37:18 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 13:37:18 +0100 (CET) Subject: [pypy-svn] r79917 - pypy/extradoc/sprintinfo/leysin-winter-2011 Message-ID: <20101209123718.47CE9282BDC@codespeak.net> Author: arigo Date: Thu Dec 9 13:37:16 2010 New Revision: 79917 Added: pypy/extradoc/sprintinfo/leysin-winter-2011/ - copied from r79817, pypy/extradoc/sprintinfo/leysin-winter-2010/ Removed: pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Log: Start a people.txt. Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2010/people.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Thu Dec 9 13:37:16 2010 @@ -1,5 +1,5 @@ -People coming to the Leysin sprint Winter 2010 +People coming to the Leysin sprint Winter 2011 ================================================== People who have a ``?`` in their arrive/depart or accomodation From david at codespeak.net Thu Dec 9 13:43:59 2010 From: david at codespeak.net (david at codespeak.net) Date: Thu, 9 Dec 2010 13:43:59 +0100 (CET) Subject: [pypy-svn] r79918 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101209124359.9FA69282BDC@codespeak.net> Author: david Date: Thu Dec 9 13:43:57 2010 New Revision: 79918 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: (arigo, david) fix copying of unicode strings Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Thu Dec 9 13:43:57 2010 @@ -741,8 +741,8 @@ bytes_loc = regalloc.force_allocate_reg(bytes_box, forbidden_vars) scale = self._get_unicode_item_scale() assert length_loc.is_reg() - self.mc.MOV_rr(bytes_loc.value, length_loc.value) - self._load_address(length_loc, 0, scale, bytes_loc) + self.mc.MOV_ri(r.ip.value, 1< 0: - self.mc.LSL_ri(r.ip.value, sizereg.value, scale) - else: - self.mc.MOV_rr(r.ip.value, sizereg.value) - self.mc.ADD_rr(result.value, result.value, r.ip.value) - self.mc.ADD_ri(result.value, result.value, baseofs) def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode): cpu = self.cpu @@ -778,7 +765,21 @@ self.cpu.translate_support_code) assert itemsize == 1 scale = 0 - self._load_address(ofsloc, ofs_items, scale, resloc, baseloc) + self._gen_address(ofsloc, ofs_items, scale, resloc, baseloc) + + def _gen_address(self, sizereg, baseofs, scale, result, baseloc=None): + assert sizereg.is_reg() + if scale > 0: + scaled_loc = r.ip + self.mc.LSL_ri(r.ip.value, sizereg.value, scale) + else: + scaled_loc = sizereg + if baseloc is not None: + assert baseloc.is_reg() + self.mc.ADD_rr(result.value, baseloc.value, scaled_loc.value) + self.mc.ADD_ri(result.value, result.value, baseofs) + else: + self.mc.ADD_ri(result.value, scaled_loc.value, baseofs) def _get_unicode_item_scale(self): _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, From arigo at codespeak.net Thu Dec 9 13:55:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 13:55:15 +0100 (CET) Subject: [pypy-svn] r79919 - pypy/extradoc/sprintinfo/leysin-winter-2011 Message-ID: <20101209125515.42B3F36C21D@codespeak.net> Author: arigo Date: Thu Dec 9 13:55:12 2010 New Revision: 79919 Added: pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt Log: Announcement. Added: pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt ============================================================================== --- (empty file) +++ pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt Thu Dec 9 13:55:12 2010 @@ -0,0 +1,68 @@ +===================================================================== + PyPy Leysin Winter Sprint (16-22nd January 2011) +===================================================================== + +The next PyPy sprint will be in Leysin, Switzerland, for the +seventh time. This is a fully public sprint: newcomers and topics +other than those proposed below are welcome. + +------------------------------ +Goals and topics of the sprint +------------------------------ + +* Now that we have released 1.4, and plan to release 1.4.1 soon + (possibly before the sprint), the sprint itself is going to be + mainly working on fixing issues reported by various users. Of + course this does not prevent people from showing up with a more + precise interest in mind. + +* We will also work on polishing and merging the long-standing + branches that are around, which could eventually lead to the + next PyPy release. These branches are notably: + + - fast-forward (Python 2.7 support, by Benjamin, Amaury, and others) + - jit-unroll-loops (improve JITting of smaller loops, by Hakan) + - arm-backend (a JIT backend for ARM, by David) + - jitypes2 (fast ctypes calls with the JIT, by Antonio). + +* And as usual, the main side goal is to have fun in winter sports :-) + We can take a day off for ski. + +----------- +Exact times +----------- + +The work days should be 16-22 January 2011. People may arrive on +the 15th already and/or leave on the 23rd. + +----------------------- +Location & Accomodation +----------------------- + +Leysin, Switzerland, "same place as before". Let me refresh your +memory: both the sprint venue and the lodging will be in a very spacious +pair of chalets built specifically for bed & breakfast: +http://www.ermina.ch/. The place has a good ADSL Internet connexion +with wireless installed. You can of course arrange your own +lodging anywhere (so long as you are in Leysin, you cannot be more than a +15 minutes walk away from the sprint venue), but I definitely recommend +lodging there too -- you won't find a better view anywhere else (though you +probably won't get much worse ones easily, either :-) + +Please *confirm* that you are coming so that we can adjust the reservations +as appropriate. The rate so far has been around 60 CHF a night all included +in 2-person rooms, with breakfast. There are larger rooms too (less +expensive) and maybe the possibility to get a single room if you really want +to. + +Please register by svn: + + http://codespeak.net/svn/pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt + +or on the pypy-sprint mailing list if you do not yet have check-in rights: + + http://codespeak.net/mailman/listinfo/pypy-sprint + +You need a Swiss-to-(insert country here) power adapter. There will be +some Swiss-to-EU adapters around -- bring a EU-format power strip if you +have one. From antocuni at codespeak.net Thu Dec 9 14:01:28 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 9 Dec 2010 14:01:28 +0100 (CET) Subject: [pypy-svn] r79920 - pypy/extradoc/sprintinfo/leysin-winter-2011 Message-ID: <20101209130128.654AE282B9D@codespeak.net> Author: antocuni Date: Thu Dec 9 14:01:26 2010 New Revision: 79920 Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt Log: typo Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt Thu Dec 9 14:01:26 2010 @@ -44,7 +44,7 @@ pair of chalets built specifically for bed & breakfast: http://www.ermina.ch/. The place has a good ADSL Internet connexion with wireless installed. You can of course arrange your own -lodging anywhere (so long as you are in Leysin, you cannot be more than a +lodging anywhere (as long as you are in Leysin, you cannot be more than a 15 minutes walk away from the sprint venue), but I definitely recommend lodging there too -- you won't find a better view anywhere else (though you probably won't get much worse ones easily, either :-) From antocuni at codespeak.net Thu Dec 9 14:03:44 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Thu, 9 Dec 2010 14:03:44 +0100 (CET) Subject: [pypy-svn] r79921 - pypy/extradoc/sprintinfo/leysin-winter-2011 Message-ID: <20101209130344.5FDB4282B9D@codespeak.net> Author: antocuni Date: Thu Dec 9 14:03:42 2010 New Revision: 79921 Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Log: add myself Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Thu Dec 9 14:03:42 2010 @@ -11,6 +11,7 @@ Name Arrive/Depart Accomodation ==================== ============== ======================= Armin Rigo -- private +Antonio Cuni 15/22 ermina ==================== ============== ======================= @@ -20,7 +21,6 @@ Name Arrive/Depart Accomodation ==================== ============== ===================== Carl Friedrich Bolz ? ? -Antonio Cuni ? ? Samuele Pedroni ? ? Anders Hammarquist ? ? Christian Tismer ? ? From arigo at codespeak.net Thu Dec 9 14:04:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:04:00 +0100 (CET) Subject: [pypy-svn] r79922 - pypy/extradoc/sprintinfo/leysin-winter-2011 Message-ID: <20101209130400.1F609282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:03:58 2010 New Revision: 79922 Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt Log: Add. Thanks antocuni Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2011/announcement.txt Thu Dec 9 14:03:58 2010 @@ -14,7 +14,8 @@ (possibly before the sprint), the sprint itself is going to be mainly working on fixing issues reported by various users. Of course this does not prevent people from showing up with a more - precise interest in mind. + precise interest in mind. If there are newcomers, we will gladly + give introduction talks. * We will also work on polishing and merging the long-standing branches that are around, which could eventually lead to the From arigo at codespeak.net Thu Dec 9 14:04:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:04:37 +0100 (CET) Subject: [pypy-svn] r79923 - pypy/extradoc/sprintinfo/leysin-winter-2011 Message-ID: <20101209130437.F0924282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:04:35 2010 New Revision: 79923 Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Log: List myself as departing the 23rd too. Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Thu Dec 9 14:04:35 2010 @@ -10,7 +10,7 @@ ==================== ============== ======================= Name Arrive/Depart Accomodation ==================== ============== ======================= -Armin Rigo -- private +Armin Rigo --/23 private Antonio Cuni 15/22 ermina ==================== ============== ======================= From arigo at codespeak.net Thu Dec 9 14:14:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:14:00 +0100 (CET) Subject: [pypy-svn] r79924 - in pypy/trunk/pypy: rlib rpython/lltypesystem rpython/lltypesystem/test rpython/memory/gc Message-ID: <20101209131400.4C4A4282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:13:58 2010 New Revision: 79924 Modified: pypy/trunk/pypy/rlib/rmmap.py pypy/trunk/pypy/rpython/lltypesystem/llarena.py pypy/trunk/pypy/rpython/lltypesystem/lltype.py pypy/trunk/pypy/rpython/lltypesystem/test/test_llarena.py pypy/trunk/pypy/rpython/memory/gc/base.py pypy/trunk/pypy/rpython/memory/gc/minimark.py Log: Merge branch/gc-debug, adding PYPY_GC_DEBUG to the runtime environment variables (setting gc.DEBUG) and tweaking PYPY_GC_NURSERY for small sizes. Also new: in gc.DEBUG mode, rotate through 23 nurseries, marking the unused 22 ones with PROT_NONE. This should help noticing when the code uses by mistake a pointer without having it listed by trackgcroot. Modified: pypy/trunk/pypy/rlib/rmmap.py ============================================================================== --- pypy/trunk/pypy/rlib/rmmap.py (original) +++ pypy/trunk/pypy/rlib/rmmap.py Thu Dec 9 14:13:58 2010 @@ -67,7 +67,7 @@ constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE', - 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE'] + 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS'] for name in constant_names: setattr(CConfig, name, rffi_platform.ConstantInteger(name)) Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/llarena.py Thu Dec 9 14:13:58 2010 @@ -26,6 +26,7 @@ self.objectptrs = {} # {offset: ptr-to-container} self.objectsizes = {} # {offset: size} self.freed = False + self.protect_inaccessible = None self.reset(zero) def __repr__(self): @@ -59,6 +60,8 @@ def check(self): if self.freed: raise ArenaError("arena was already freed") + if self.protect_inaccessible is not None: + raise ArenaError("arena is currently arena_protect()ed") def _getid(self): address, length = self.usagemap.buffer_info() @@ -127,6 +130,21 @@ def mark_freed(self): self.freed = True # this method is a hook for tests + def set_protect(self, inaccessible): + if inaccessible: + assert self.protect_inaccessible is None + saved = [] + for ptr in self.objectptrs.values(): + obj = ptr._obj + saved.append((obj, obj._protect())) + self.protect_inaccessible = saved + else: + assert self.protect_inaccessible is not None + saved = self.protect_inaccessible + for obj, storage in saved: + obj._unprotect(storage) + self.protect_inaccessible = None + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -365,6 +383,16 @@ """ return Arena(ptr.arena.nbytes, False).getaddr(0) +def arena_protect(arena_addr, size, inaccessible): + """For debugging, set or reset memory protection on an arena. + For now, the starting point and size should reference the whole arena. + The value of 'inaccessible' is a boolean. + """ + arena_addr = getfakearenaaddress(arena_addr) + assert arena_addr.offset == 0 + assert size == arena_addr.arena.nbytes + arena_addr.arena.set_protect(inaccessible) + # ____________________________________________________________ # # Translation support: the functions above turn into the code below. @@ -475,6 +503,44 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear +if os.name == "posix": + from pypy.translator.tool.cbuild import ExternalCompilationInfo + _eci = ExternalCompilationInfo(includes=['sys/mman.h']) + raw_mprotect = rffi.llexternal('mprotect', + [llmemory.Address, rffi.SIZE_T, rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True, + compilation_info=_eci) + def llimpl_protect(addr, size, inaccessible): + if inaccessible: + prot = 0 + else: + from pypy.rlib.rmmap import PROT_READ, PROT_WRITE + prot = PROT_READ | PROT_WRITE + raw_mprotect(addr, rffi.cast(rffi.SIZE_T, size), + rffi.cast(rffi.INT, prot)) + # ignore potential errors + has_protect = True + +elif os.name == 'nt': + def llimpl_protect(addr, size, inaccessible): + from pypy.rlib.rmmap import VirtualProtect, LPDWORD + if inaccessible: + from pypy.rlib.rmmap import PAGE_NOACCESS as newprotect + else: + from pypy.rlib.rmmap import PAGE_READWRITE as newprotect + arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') + VirtualProtect(rffi.cast(rffi.VOIDP, addr), + rffi.cast(rffi.SIZE_T, size), + newprotect, + arg) + # ignore potential errors + lltype.free(arg, flavor='raw') + has_protect = True + +else: + has_protect = False + llimpl_malloc = rffi.llexternal('malloc', [lltype.Signed], llmemory.Address, sandboxsafe=True, _nowrapper=True) @@ -544,6 +610,21 @@ 'll_arena.arena_new_view', llimpl=llimpl_arena_new_view, llfakeimpl=arena_new_view, sandboxsafe=True) +def llimpl_arena_protect(addr, size, inaccessible): + if has_protect: + # do some alignment + start = rffi.cast(lltype.Signed, addr) + end = start + size + start = (start + 4095) & ~ 4095 + end = end & ~ 4095 + if end > start: + llimpl_protect(rffi.cast(llmemory.Address, start), end-start, + inaccessible) +register_external(arena_protect, [llmemory.Address, lltype.Signed, + lltype.Bool], lltype.Void, + 'll_arena.arena_protect', llimpl=llimpl_arena_protect, + llfakeimpl=arena_protect, sandboxsafe=True) + def llimpl_getfakearenaaddress(addr): return addr register_external(getfakearenaaddress, [llmemory.Address], llmemory.Address, Modified: pypy/trunk/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lltype.py Thu Dec 9 14:13:58 2010 @@ -1381,6 +1381,15 @@ self._check() # no double-frees self._storage = None + def _protect(self): + result = self._storage + self._free() # no double-frees or double-protects + return result + + def _unprotect(self, saved_storage): + assert self._storage is None + self._storage = saved_storage + def _was_freed(self): if self._storage is None: return True Modified: pypy/trunk/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/test/test_llarena.py Thu Dec 9 14:13:58 2010 @@ -6,6 +6,8 @@ from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view from pypy.rpython.lltypesystem.llarena import arena_shrink_obj +from pypy.rpython.lltypesystem.llarena import arena_protect, has_protect +from pypy.translator.c.test import test_genc, test_standalone def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -265,8 +267,7 @@ assert res == 42 def test_compiled(): - from pypy.translator.c.test.test_genc import compile - fn = compile(test_look_inside_object, []) + fn = test_genc.compile(test_look_inside_object, []) res = fn() assert res == 42 @@ -282,3 +283,51 @@ arena_reserve(a, size_gc_header + llmemory.sizeof(S, 10)) arena_shrink_obj(a, size_gc_header + llmemory.sizeof(S, 5)) arena_reset(a, size_gc_header + llmemory.sizeof(S, 5), False) + +def test_arena_protect(): + a = arena_malloc(100, False) + S = lltype.Struct('S', ('x', lltype.Signed)) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 100, True) + py.test.raises(ArenaError, arena_reserve, a + 48, llmemory.sizeof(S)) + py.test.raises(RuntimeError, "p.x") + py.test.raises(RuntimeError, "p.x = 124") + arena_protect(a, 100, False) + assert p.x == 123 + p.x = 125 + assert p.x == 125 + + +class TestStandalone(test_standalone.StandaloneTests): + def test_compiled_arena_protect(self): + import os + from pypy.translator.c.test.test_genc import compile + S = lltype.Struct('S', ('x', lltype.Signed)) + # + def fn(argv): + testrun = int(argv[1]) + a = arena_malloc(65536, False) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a + 23432, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 65536, True) + result = 0 + if testrun == 1: + print p.x # segfault + if testrun == 2: + p.x = 124 # segfault + arena_protect(a, 65536, False) + p.x += 10 + print p.x + return 0 + # + t, cbuilder = self.compile(fn) + data = cbuilder.cmdexec('0') + assert data == '133\n' + if has_protect: + cbuilder.cmdexec('1', expect_crash=True) + cbuilder.cmdexec('2', expect_crash=True) Modified: pypy/trunk/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/base.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/base.py Thu Dec 9 14:13:58 2010 @@ -40,8 +40,7 @@ # More stuff that needs to be initialized when the GC is already # fully working. (Only called by gctransform/framework for now.) from pypy.rpython.memory.gc import env - if env.read_from_env('PYPY_GC_DEBUG') > 0: - self.DEBUG = True + self.DEBUG = env.read_from_env('PYPY_GC_DEBUG') def _teardown(self): pass Modified: pypy/trunk/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/minimark.py Thu Dec 9 14:13:58 2010 @@ -3,7 +3,8 @@ Environment variables can be used to fine-tune the following parameters: PYPY_GC_NURSERY The nursery size. Defaults to half the size of - the L2 cache. Try values like '1.2MB'. + the L2 cache. Try values like '1.2MB'. Small values + (like 1 or 1KB) are useful for debugging. PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', which means trigger a major collection when the @@ -34,7 +35,9 @@ times the nursery. PYPY_GC_DEBUG Enable extra checks around collections that are - too slow for normal use. + too slow for normal use. Values are 0 (off), + 1 (on major collections) or 2 (also on minor + collections). """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -226,7 +229,8 @@ self.nursery = NULL self.nursery_free = NULL self.nursery_top = NULL - self.debug_always_do_minor_collect = False + self.debug_tiny_nursery = -1 + self.debug_rotating_nurseries = None # # The ArenaCollection() handles the nonmovable objects allocation. if ArenaCollectionClass is None: @@ -294,15 +298,20 @@ # From there on, the GC is fully initialized and the code # below can use it newsize = env.read_from_env('PYPY_GC_NURSERY') - # PYPY_GC_NURSERY=1 forces a minor collect for every malloc. - # Useful to debug external factors, like trackgcroot or the - # handling of the write barrier. - self.debug_always_do_minor_collect = newsize == 1 + # PYPY_GC_NURSERY=smallvalue means that minor collects occur + # very frequently; the extreme case is PYPY_GC_NURSERY=1, which + # forces a minor collect for every malloc. Useful to debug + # external factors, like trackgcroot or the handling of the write + # barrier. Implemented by still using 'minsize' for the nursery + # size (needed to handle e.g. mallocs of 8249 words) but hacking + # at the current nursery position in collect_and_reserve(). if newsize <= 0: newsize = env.estimate_best_nursery_size() if newsize <= 0: newsize = defaultsize - newsize = max(newsize, minsize) + if newsize < minsize: + self.debug_tiny_nursery = newsize & ~(WORD-1) + newsize = minsize # major_coll = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT') if major_coll > 1.0: @@ -335,17 +344,24 @@ self.allocate_nursery() - def allocate_nursery(self): - debug_start("gc-set-nursery-size") - debug_print("nursery size:", self.nursery_size) + def _nursery_memory_size(self): + extra = self.nonlarge_gcptrs_max + 1 + return self.nursery_size + extra + + def _alloc_nursery(self): # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize_clear(). The few extra pages are never used # anyway so it doesn't even count. - extra = self.nonlarge_gcptrs_max + 1 - self.nursery = llarena.arena_malloc(self.nursery_size + extra, 2) - if not self.nursery: + nursery = llarena.arena_malloc(self._nursery_memory_size(), 2) + if not nursery: raise MemoryError("cannot allocate nursery") + return nursery + + def allocate_nursery(self): + debug_start("gc-set-nursery-size") + debug_print("nursery size:", self.nursery_size) + self.nursery = self._alloc_nursery() # the current position in the nursery: self.nursery_free = self.nursery # the end of the nursery: @@ -379,6 +395,39 @@ return bounded + def post_setup(self): + # set up extra stuff for PYPY_GC_DEBUG. + MovingGCBase.post_setup(self) + if self.DEBUG and llarena.has_protect: + # gc debug mode: allocate 23 nurseries instead of just 1, + # and use them alternatively, while mprotect()ing the unused + # ones to detect invalid access. + debug_start("gc-debug") + self.debug_rotating_nurseries = [] + for i in range(22): + nurs = self._alloc_nursery() + llarena.arena_protect(nurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(nurs) + debug_print("allocated", len(self.debug_rotating_nurseries), + "extra nurseries") + debug_stop("gc-debug") + + def debug_rotate_nursery(self): + if self.debug_rotating_nurseries is not None: + debug_start("gc-debug") + oldnurs = self.nursery + llarena.arena_protect(oldnurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(oldnurs) + # + newnurs = self.debug_rotating_nurseries.pop(0) + llarena.arena_protect(newnurs, self._nursery_memory_size(), False) + self.nursery = newnurs + self.nursery_top = self.nursery + self.nursery_size + debug_print("switching to nursery", self.nursery, + "size", self.nursery_size) + debug_stop("gc-debug") + + def malloc_fixedsize_clear(self, typeid, size, can_collect=True, needs_finalizer=False, contains_weakptr=False): ll_assert(can_collect, "!can_collect") @@ -516,8 +565,9 @@ self.nursery_free = result + totalsize ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow") # - if self.debug_always_do_minor_collect: - self.nursery_free = self.nursery_top + if self.debug_tiny_nursery >= 0: # for debugging + if self.nursery_top - self.nursery_free > self.debug_tiny_nursery: + self.nursery_free = self.nursery_top - self.debug_tiny_nursery # return result collect_and_reserve._dont_inline_ = True @@ -1003,13 +1053,14 @@ # All live nursery objects are out, and the rest dies. Fill # the whole nursery with zero and reset the current nursery pointer. llarena.arena_reset(self.nursery, self.nursery_size, 2) + self.debug_rotate_nursery() self.nursery_free = self.nursery # debug_print("minor collect, total memory used:", self.get_total_memory_used()) + if self.DEBUG >= 2: + self.debug_check_consistency() # expensive! debug_stop("gc-minor") - if 0: # not we_are_translated(): - self.debug_check_consistency() # xxx expensive! def collect_roots_in_nursery(self): From arigo at codespeak.net Thu Dec 9 14:15:14 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:15:14 +0100 (CET) Subject: [pypy-svn] r79925 - pypy/trunk/lib-python/modified-2.5.2/encodings Message-ID: <20101209131514.B2B20282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:15:12 2010 New Revision: 79925 Removed: pypy/trunk/lib-python/modified-2.5.2/encodings/ Log: I think that all changes done to the "encodings" package can be safely reverted now. From arigo at codespeak.net Thu Dec 9 14:24:12 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:24:12 +0100 (CET) Subject: [pypy-svn] r79926 - pypy/trunk/pypy/interpreter/pyparser/test Message-ID: <20101209132412.4064E282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:24:10 2010 New Revision: 79926 Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py Log: Fix test. Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py Thu Dec 9 14:24:10 2010 @@ -75,9 +75,10 @@ for input in ("())", "(()", "((", "))"): py.test.raises(SyntaxError, parse, input) exc = py.test.raises(SyntaxError, parse, "x = (\n\n(),\n(),").value - assert exc.msg == "EOF in multi-line statement" + assert exc.msg == "parenthesis is never closed" assert exc.lineno == 1 assert exc.offset == 5 + assert exc.lastlineno == 5 def test_is(self): self.parse("x is y") From arigo at codespeak.net Thu Dec 9 14:25:57 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:25:57 +0100 (CET) Subject: [pypy-svn] r79927 - pypy/trunk/pypy/translator/sandbox/test Message-ID: <20101209132557.46684282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:25:55 2010 New Revision: 79927 Modified: pypy/trunk/pypy/translator/sandbox/test/test_sandbox.py Log: Fix test. Modified: pypy/trunk/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/trunk/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/trunk/pypy/translator/sandbox/test/test_sandbox.py Thu Dec 9 14:25:55 2010 @@ -148,6 +148,7 @@ if sys.platform == 'linux2': # on Mac, uses another (sandboxsafe) approach expect(f, g, "ll_os.ll_os_open", ("/proc/cpuinfo", 0, 420), OSError(5232, "xyz")) + expect(f, g, "ll_os.ll_os_getenv", ("PYPY_GC_DEBUG",), None) g.close() tail = f.read() f.close() From arigo at codespeak.net Thu Dec 9 14:28:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:28:19 +0100 (CET) Subject: [pypy-svn] r79928 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20101209132819.3ADA5282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:28:17 2010 New Revision: 79928 Added: pypy/trunk/lib-python/modified-2.5.2/test/test_eof.py - copied, changed from r79924, pypy/trunk/lib-python/2.5.2/test/test_eof.py Log: Fix this test. Too precise... Copied: pypy/trunk/lib-python/modified-2.5.2/test/test_eof.py (from r79924, pypy/trunk/lib-python/2.5.2/test/test_eof.py) ============================================================================== --- pypy/trunk/lib-python/2.5.2/test/test_eof.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_eof.py Thu Dec 9 14:28:17 2010 @@ -17,7 +17,7 @@ raise test_support.TestFailed def test_EOFS(self): - expect = "EOF while scanning triple-quoted string (, line 1)" + expect = "EOF while scanning triple-quoted string (, lines 1-2)" try: eval("""'''this is a test""") except SyntaxError, msg: From arigo at codespeak.net Thu Dec 9 14:30:25 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:30:25 +0100 (CET) Subject: [pypy-svn] r79929 - pypy/trunk/lib-python/modified-2.5.2/test Message-ID: <20101209133025.11E39282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:30:24 2010 New Revision: 79929 Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_genexps.py Log: Fix test. Modified: pypy/trunk/lib-python/modified-2.5.2/test/test_genexps.py ============================================================================== --- pypy/trunk/lib-python/modified-2.5.2/test/test_genexps.py (original) +++ pypy/trunk/lib-python/modified-2.5.2/test/test_genexps.py Thu Dec 9 14:30:24 2010 @@ -77,10 +77,10 @@ Verify that parenthesis are required in a statement >>> def f(n): - ... return i*i for i in xrange(n) + ... return i*i for i in xrange(n) #doctest: +ELLIPSIS Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax... Verify that parenthesis are required when used as a keyword argument value From arigo at codespeak.net Thu Dec 9 14:50:38 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 14:50:38 +0100 (CET) Subject: [pypy-svn] r79930 - pypy/branch/gc-debug Message-ID: <20101209135038.BAF2C282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 14:50:37 2010 New Revision: 79930 Removed: pypy/branch/gc-debug/ Log: Remove merged branch. From danchr at codespeak.net Thu Dec 9 14:59:30 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Thu, 9 Dec 2010 14:59:30 +0100 (CET) Subject: [pypy-svn] r79931 - pypy/branch/fast-forward-darwin Message-ID: <20101209135930.A5977282BAD@codespeak.net> Author: danchr Date: Thu Dec 9 14:59:28 2010 New Revision: 79931 Added: pypy/branch/fast-forward-darwin/ (props changed) - copied from r79930, pypy/branch/fast-forward/ Log: Branch for making fast-forward work on Darwin. From david at codespeak.net Thu Dec 9 15:04:58 2010 From: david at codespeak.net (david at codespeak.net) Date: Thu, 9 Dec 2010 15:04:58 +0100 (CET) Subject: [pypy-svn] r79932 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101209140458.EB550282B9D@codespeak.net> Author: david Date: Thu Dec 9 15:04:57 2010 New Revision: 79932 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: Fix some more allocation issues in copy operations Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Thu Dec 9 15:04:57 2010 @@ -710,13 +710,19 @@ regalloc.possibly_free_var(args[2]) # it if ==args[3] or args[4] srcaddr_box = TempBox() forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = regalloc.force_allocate_reg(srcaddr_box, forbidden_vars) + srcaddr_loc = regalloc.force_allocate_reg(srcaddr_box, + forbidden_vars, selected_reg=r.r1) self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, is_unicode=is_unicode) # compute the destination address + forbidden_vars = [args[4], args[3], srcaddr_box] + dstaddr_box = TempBox() + dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, selected_reg=r.r0) + forbidden_vars.append(dstaddr_box) base_loc, box = self._ensure_value_is_boxed(args[1], regalloc, forbidden_vars) args.append(box) + forbidden_vars.append(box) ofs_loc, box = self._ensure_value_is_boxed(args[3], regalloc, forbidden_vars) args.append(box) assert base_loc.is_reg() @@ -724,10 +730,6 @@ regalloc.possibly_free_var(args[1]) if args[3] is not args[4]: # more of the MESS described above regalloc.possibly_free_var(args[3]) - forbidden_vars = [args[4], srcaddr_box] - dstaddr_box = TempBox() - dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, - forbidden_vars) self._gen_address_inside_string(base_loc, ofs_loc, dstaddr_loc, is_unicode=is_unicode) From arigo at codespeak.net Thu Dec 9 15:08:53 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 15:08:53 +0100 (CET) Subject: [pypy-svn] r79933 - in pypy/trunk/pypy: interpreter/pyparser/test interpreter/test module/exceptions Message-ID: <20101209140853.937A0282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 15:08:51 2010 New Revision: 79933 Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py pypy/trunk/pypy/interpreter/test/test_compiler.py pypy/trunk/pypy/module/exceptions/interp_exceptions.py Log: Fix for the stdlib's test_traceback: make sure that SyntaxError().args[1] is a 4-tuple, as in CPython, and not a 5-tuple. Apparently traceback.format_exception_only() is sensible to that. Modified: pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py ============================================================================== --- pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py (original) +++ pypy/trunk/pypy/interpreter/pyparser/test/test_pyparse.py Thu Dec 9 15:08:51 2010 @@ -100,6 +100,7 @@ input = "def f():\n pass\n next_stmt" exc = py.test.raises(IndentationError, parse, input).value assert exc.msg == "unindent does not match any outer indentation level" + assert exc.lineno == 3 def test_mac_newline(self): self.parse("this_is\ra_mac\rfile") Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/test/test_compiler.py Thu Dec 9 15:08:51 2010 @@ -68,6 +68,17 @@ space.raises_w(space.w_SyntaxError, self.compiler.compile_command, 'if 1:\n x\n y\n', '?', 'exec', 0) + def test_syntaxerror_attrs(self): + w_args = self.space.appexec([], r"""(): + try: + exec 'if 1:\n x\n y\n' + except SyntaxError, e: + return e.args + """) + assert self.space.unwrap(w_args) == ( + 'unindent does not match any outer indentation level', + (None, 3, 0, ' y\n')) + def test_getcodeflags(self): code = self.compiler.compile('from __future__ import division\n', '', 'exec', 0) Modified: pypy/trunk/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/trunk/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/trunk/pypy/module/exceptions/interp_exceptions.py Thu Dec 9 15:08:51 2010 @@ -465,7 +465,11 @@ if len(values_w) > 1: self.w_lineno = values_w[1] if len(values_w) > 2: self.w_offset = values_w[2] if len(values_w) > 3: self.w_text = values_w[3] - if len(values_w) > 4: self.w_lastlineno = values_w[4] + if len(values_w) > 4: + self.w_lastlineno = values_w[4] # PyPy extension + # kill the extra items from args_w to prevent undesired effects + args_w = args_w[:] + args_w[1] = space.newtuple(values_w[:4]) W_BaseException.descr_init(self, space, args_w) descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] From david at codespeak.net Thu Dec 9 15:45:03 2010 From: david at codespeak.net (david at codespeak.net) Date: Thu, 9 Dec 2010 15:45:03 +0100 (CET) Subject: [pypy-svn] r79934 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101209144503.54801282B9D@codespeak.net> Author: david Date: Thu Dec 9 15:45:02 2010 New Revision: 79934 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Log: Another fix for string copying Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Thu Dec 9 15:45:02 2010 @@ -710,8 +710,7 @@ regalloc.possibly_free_var(args[2]) # it if ==args[3] or args[4] srcaddr_box = TempBox() forbidden_vars = [args[1], args[3], args[4], srcaddr_box] - srcaddr_loc = regalloc.force_allocate_reg(srcaddr_box, - forbidden_vars, selected_reg=r.r1) + srcaddr_loc = regalloc.force_allocate_reg(srcaddr_box, selected_reg=r.r1) self._gen_address_inside_string(base_loc, ofs_loc, srcaddr_loc, is_unicode=is_unicode) From fijal at codespeak.net Thu Dec 9 16:34:16 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 9 Dec 2010 16:34:16 +0100 (CET) Subject: [pypy-svn] r79935 - in pypy/branch/out-of-line-guards: . ctypes_configure lib-python lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/distutils/tests lib-python/modified-2.5.2/encodings lib-python/modified-2.5.2/test lib-python/modified-2.5.2/test/output lib_pypy lib_pypy/_ctypes lib_pypy/pypy_test pypy pypy/annotation pypy/annotation/test pypy/config pypy/doc pypy/doc/config pypy/doc/statistic pypy/interpreter pypy/interpreter/pyparser pypy/interpreter/pyparser/test pypy/interpreter/test pypy/jit pypy/jit/backend pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/optimizeopt pypy/jit/metainterp/test pypy/jit/tl pypy/jit/tl/spli pypy/jit/tool pypy/jit/tool/test pypy/module/__pypy__ pypy/module/__pypy__/test pypy/module/_lsprof pypy/module/_lsprof/test pypy/module/_minimal_curses pypy/module/_pickle_support pypy/module/_stackless pypy/module/array/benchmark pypy/module/array/test pypy/module/binascii pypy/module/binascii/test pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/src pypy/module/cpyext/test pypy/module/exceptions pypy/module/fcntl/test pypy/module/imp pypy/module/itertools pypy/module/posix pypy/module/posix/test pypy/module/pyexpat pypy/module/pyexpat/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/sys pypy/module/sys/test pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/rsre pypy/rlib/rsre/test pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/memory/gc pypy/rpython/memory/gc/test pypy/rpython/memory/gctransform pypy/rpython/module pypy/rpython/ootypesystem pypy/rpython/test pypy/tool pypy/tool/release pypy/tool/release/test pypy/translator pypy/translator/c pypy/translator/c/gcc pypy/translator/c/gcc/test pypy/translator/c/src pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/platform pypy/translator/sandbox/test pypy/translator/tool site-packages Message-ID: <20101209153416.72A4C282B9D@codespeak.net> Author: fijal Date: Thu Dec 9 16:33:59 2010 New Revision: 79935 Added: pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/test_eof.py - copied unchanged from r79934, pypy/trunk/lib-python/modified-2.5.2/test/test_eof.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/codebuf.py - copied unchanged from r79934, pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/branch/out-of-line-guards/pypy/module/binascii/ (props changed) - copied from r79934, pypy/trunk/pypy/module/binascii/ Removed: pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/encodings/ pypy/branch/out-of-line-guards/pypy/doc/config/translation.jit_debug.txt pypy/branch/out-of-line-guards/pypy/translator/platform/freebsd7.py Modified: pypy/branch/out-of-line-guards/ (props changed) pypy/branch/out-of-line-guards/ctypes_configure/configure.py pypy/branch/out-of-line-guards/lib-python/conftest.py pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/msvccompiler.py pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/unixccompiler.py pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/output/test_cProfile pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/test_genexps.py pypy/branch/out-of-line-guards/lib_pypy/_ctypes/array.py pypy/branch/out-of-line-guards/lib_pypy/_ctypes/basics.py pypy/branch/out-of-line-guards/lib_pypy/_ctypes/builtin.py pypy/branch/out-of-line-guards/lib_pypy/_ctypes/function.py pypy/branch/out-of-line-guards/lib_pypy/_ctypes/pointer.py pypy/branch/out-of-line-guards/lib_pypy/_ctypes/primitive.py pypy/branch/out-of-line-guards/lib_pypy/_ctypes/structure.py pypy/branch/out-of-line-guards/lib_pypy/_hashlib.py pypy/branch/out-of-line-guards/lib_pypy/_locale.py pypy/branch/out-of-line-guards/lib_pypy/_marshal.py pypy/branch/out-of-line-guards/lib_pypy/_minimal_curses.py pypy/branch/out-of-line-guards/lib_pypy/_pypy_interact.py pypy/branch/out-of-line-guards/lib_pypy/binascii.py pypy/branch/out-of-line-guards/lib_pypy/cPickle.py pypy/branch/out-of-line-guards/lib_pypy/cmath.py pypy/branch/out-of-line-guards/lib_pypy/ctypes_support.py pypy/branch/out-of-line-guards/lib_pypy/grp.py pypy/branch/out-of-line-guards/lib_pypy/hashlib.py pypy/branch/out-of-line-guards/lib_pypy/itertools.py pypy/branch/out-of-line-guards/lib_pypy/msvcrt.py pypy/branch/out-of-line-guards/lib_pypy/pwd.py pypy/branch/out-of-line-guards/lib_pypy/pyexpat.py pypy/branch/out-of-line-guards/lib_pypy/pypy_test/test_hashlib.py pypy/branch/out-of-line-guards/lib_pypy/pypy_test/test_structseq.py pypy/branch/out-of-line-guards/lib_pypy/readline.py pypy/branch/out-of-line-guards/lib_pypy/resource.py pypy/branch/out-of-line-guards/lib_pypy/syslog.py pypy/branch/out-of-line-guards/pypy/ (props changed) pypy/branch/out-of-line-guards/pypy/annotation/annrpython.py pypy/branch/out-of-line-guards/pypy/annotation/binaryop.py pypy/branch/out-of-line-guards/pypy/annotation/bookkeeper.py pypy/branch/out-of-line-guards/pypy/annotation/description.py pypy/branch/out-of-line-guards/pypy/annotation/listdef.py pypy/branch/out-of-line-guards/pypy/annotation/model.py pypy/branch/out-of-line-guards/pypy/annotation/specialize.py pypy/branch/out-of-line-guards/pypy/annotation/test/test_annrpython.py pypy/branch/out-of-line-guards/pypy/annotation/unaryop.py pypy/branch/out-of-line-guards/pypy/config/pypyoption.py pypy/branch/out-of-line-guards/pypy/config/translationoption.py pypy/branch/out-of-line-guards/pypy/conftest.py pypy/branch/out-of-line-guards/pypy/doc/cpython_differences.txt pypy/branch/out-of-line-guards/pypy/doc/faq.txt pypy/branch/out-of-line-guards/pypy/doc/index.txt pypy/branch/out-of-line-guards/pypy/doc/release-1.4.0.txt pypy/branch/out-of-line-guards/pypy/doc/sprint-reports.txt pypy/branch/out-of-line-guards/pypy/doc/statistic/release_dates.dat pypy/branch/out-of-line-guards/pypy/doc/statistic/sprint_dates.dat pypy/branch/out-of-line-guards/pypy/interpreter/argument.py pypy/branch/out-of-line-guards/pypy/interpreter/baseobjspace.py pypy/branch/out-of-line-guards/pypy/interpreter/executioncontext.py pypy/branch/out-of-line-guards/pypy/interpreter/function.py pypy/branch/out-of-line-guards/pypy/interpreter/gateway.py pypy/branch/out-of-line-guards/pypy/interpreter/generator.py pypy/branch/out-of-line-guards/pypy/interpreter/mixedmodule.py pypy/branch/out-of-line-guards/pypy/interpreter/pycode.py pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/error.py pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/pytokenizer.py pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/test/test_pyparse.py pypy/branch/out-of-line-guards/pypy/interpreter/test/test_argument.py pypy/branch/out-of-line-guards/pypy/interpreter/test/test_compiler.py pypy/branch/out-of-line-guards/pypy/interpreter/test/test_executioncontext.py pypy/branch/out-of-line-guards/pypy/interpreter/test/test_function.py pypy/branch/out-of-line-guards/pypy/interpreter/test/test_gateway.py pypy/branch/out-of-line-guards/pypy/jit/backend/detect_cpu.py pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/gc.py pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/llmodel.py pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/out-of-line-guards/pypy/jit/backend/model.py pypy/branch/out-of-line-guards/pypy/jit/backend/test/runner_test.py pypy/branch/out-of-line-guards/pypy/jit/backend/test/test_random.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/assembler.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/regalloc.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/regloc.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/runner.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/rx86.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/support.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_runner.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_zmath.py pypy/branch/out-of-line-guards/pypy/jit/backend/x86/valgrind.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/call.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/regalloc.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/support.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_list.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_void_list.py pypy/branch/out-of-line-guards/pypy/jit/conftest.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/jitprof.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizefindnode.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/__init__.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/optimizer.py (contents, props changed) pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_compile.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_exception.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_fficall.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_list.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_logger.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizefficall.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_recursive.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmstate.py pypy/branch/out-of-line-guards/pypy/jit/tl/spli/interpreter.py pypy/branch/out-of-line-guards/pypy/jit/tl/tl.py pypy/branch/out-of-line-guards/pypy/jit/tool/jitoutput.py pypy/branch/out-of-line-guards/pypy/jit/tool/test/test_jitoutput.py pypy/branch/out-of-line-guards/pypy/module/__pypy__/__init__.py pypy/branch/out-of-line-guards/pypy/module/__pypy__/interp_magic.py pypy/branch/out-of-line-guards/pypy/module/__pypy__/test/test_special.py pypy/branch/out-of-line-guards/pypy/module/_lsprof/interp_lsprof.py pypy/branch/out-of-line-guards/pypy/module/_lsprof/test/test_cprofile.py pypy/branch/out-of-line-guards/pypy/module/_minimal_curses/__init__.py pypy/branch/out-of-line-guards/pypy/module/_pickle_support/maker.py pypy/branch/out-of-line-guards/pypy/module/_stackless/interp_coroutine.py pypy/branch/out-of-line-guards/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/out-of-line-guards/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/out-of-line-guards/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/out-of-line-guards/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/out-of-line-guards/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/out-of-line-guards/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/out-of-line-guards/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/out-of-line-guards/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/out-of-line-guards/pypy/module/array/test/test_array.py pypy/branch/out-of-line-guards/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/out-of-line-guards/pypy/module/binascii/test/ (props changed) pypy/branch/out-of-line-guards/pypy/module/cpyext/__init__.py pypy/branch/out-of-line-guards/pypy/module/cpyext/api.py pypy/branch/out-of-line-guards/pypy/module/cpyext/cdatetime.py pypy/branch/out-of-line-guards/pypy/module/cpyext/include/Python.h pypy/branch/out-of-line-guards/pypy/module/cpyext/intobject.py pypy/branch/out-of-line-guards/pypy/module/cpyext/object.py pypy/branch/out-of-line-guards/pypy/module/cpyext/pyerrors.py pypy/branch/out-of-line-guards/pypy/module/cpyext/pythonrun.py pypy/branch/out-of-line-guards/pypy/module/cpyext/sequence.py pypy/branch/out-of-line-guards/pypy/module/cpyext/slotdefs.py pypy/branch/out-of-line-guards/pypy/module/cpyext/src/getargs.c pypy/branch/out-of-line-guards/pypy/module/cpyext/state.py pypy/branch/out-of-line-guards/pypy/module/cpyext/stubs.py pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_arraymodule.py pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_cpyext.py pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_datetime.py pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_intobject.py pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_object.py pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_pyerrors.py pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_sequence.py pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_typeobject.py pypy/branch/out-of-line-guards/pypy/module/cpyext/typeobject.py pypy/branch/out-of-line-guards/pypy/module/exceptions/interp_exceptions.py pypy/branch/out-of-line-guards/pypy/module/fcntl/test/test_fcntl.py pypy/branch/out-of-line-guards/pypy/module/imp/importing.py pypy/branch/out-of-line-guards/pypy/module/itertools/interp_itertools.py pypy/branch/out-of-line-guards/pypy/module/posix/__init__.py pypy/branch/out-of-line-guards/pypy/module/posix/interp_posix.py pypy/branch/out-of-line-guards/pypy/module/posix/test/test_posix2.py pypy/branch/out-of-line-guards/pypy/module/pyexpat/interp_pyexpat.py pypy/branch/out-of-line-guards/pypy/module/pyexpat/test/test_parser.py pypy/branch/out-of-line-guards/pypy/module/pypyjit/policy.py pypy/branch/out-of-line-guards/pypy/module/pypyjit/test/test_policy.py pypy/branch/out-of-line-guards/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/out-of-line-guards/pypy/module/sys/__init__.py pypy/branch/out-of-line-guards/pypy/module/sys/state.py pypy/branch/out-of-line-guards/pypy/module/sys/test/test_initialpath.py pypy/branch/out-of-line-guards/pypy/module/sys/vm.py pypy/branch/out-of-line-guards/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py pypy/branch/out-of-line-guards/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py pypy/branch/out-of-line-guards/pypy/objspace/std/complexobject.py pypy/branch/out-of-line-guards/pypy/objspace/std/complextype.py pypy/branch/out-of-line-guards/pypy/objspace/std/floattype.py pypy/branch/out-of-line-guards/pypy/objspace/std/longobject.py pypy/branch/out-of-line-guards/pypy/objspace/std/mapdict.py pypy/branch/out-of-line-guards/pypy/objspace/std/objspace.py pypy/branch/out-of-line-guards/pypy/objspace/std/strutil.py pypy/branch/out-of-line-guards/pypy/objspace/std/test/test_complexobject.py pypy/branch/out-of-line-guards/pypy/objspace/std/test/test_strutil.py pypy/branch/out-of-line-guards/pypy/rlib/debug.py pypy/branch/out-of-line-guards/pypy/rlib/jit.py pypy/branch/out-of-line-guards/pypy/rlib/libffi.py pypy/branch/out-of-line-guards/pypy/rlib/rarithmetic.py pypy/branch/out-of-line-guards/pypy/rlib/rdynload.py pypy/branch/out-of-line-guards/pypy/rlib/rerased.py (contents, props changed) pypy/branch/out-of-line-guards/pypy/rlib/rmmap.py pypy/branch/out-of-line-guards/pypy/rlib/rsre/rsre_core.py pypy/branch/out-of-line-guards/pypy/rlib/rsre/test/test_zjit.py pypy/branch/out-of-line-guards/pypy/rlib/test/test_debug.py pypy/branch/out-of-line-guards/pypy/rlib/test/test_libffi.py pypy/branch/out-of-line-guards/pypy/rlib/test/test_rerased.py (props changed) pypy/branch/out-of-line-guards/pypy/rpython/extfunc.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/llarena.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/llmemory.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/lltype.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rbuiltin.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rdict.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rpbc.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/test/test_llarena.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/base.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/generation.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/inspector.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/markcompact.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/minimark.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/semispace.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/boehm.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/framework.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/support.py pypy/branch/out-of-line-guards/pypy/rpython/module/ll_os.py pypy/branch/out-of-line-guards/pypy/rpython/module/ll_time.py pypy/branch/out-of-line-guards/pypy/rpython/ootypesystem/rpbc.py pypy/branch/out-of-line-guards/pypy/rpython/rlist.py pypy/branch/out-of-line-guards/pypy/rpython/rpbc.py pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py pypy/branch/out-of-line-guards/pypy/rpython/test/test_rint.py pypy/branch/out-of-line-guards/pypy/rpython/test/test_rlist.py pypy/branch/out-of-line-guards/pypy/tool/ansi_print.py pypy/branch/out-of-line-guards/pypy/tool/error.py pypy/branch/out-of-line-guards/pypy/tool/logparser.py pypy/branch/out-of-line-guards/pypy/tool/release/force-builds.py pypy/branch/out-of-line-guards/pypy/tool/release/make_release.py pypy/branch/out-of-line-guards/pypy/tool/release/package.py pypy/branch/out-of-line-guards/pypy/tool/release/test/test_package.py pypy/branch/out-of-line-guards/pypy/translator/c/funcgen.py pypy/branch/out-of-line-guards/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/out-of-line-guards/pypy/translator/c/gcc/trackgcroot.py pypy/branch/out-of-line-guards/pypy/translator/c/src/debug_alloc.h pypy/branch/out-of-line-guards/pypy/translator/c/src/debug_print.h pypy/branch/out-of-line-guards/pypy/translator/c/src/mem.h pypy/branch/out-of-line-guards/pypy/translator/c/test/test_newgc.py pypy/branch/out-of-line-guards/pypy/translator/c/test/test_standalone.py pypy/branch/out-of-line-guards/pypy/translator/driver.py pypy/branch/out-of-line-guards/pypy/translator/goal/app_main.py pypy/branch/out-of-line-guards/pypy/translator/goal/test2/test_app_main.py pypy/branch/out-of-line-guards/pypy/translator/platform/__init__.py pypy/branch/out-of-line-guards/pypy/translator/platform/darwin.py pypy/branch/out-of-line-guards/pypy/translator/platform/linux.py pypy/branch/out-of-line-guards/pypy/translator/platform/posix.py pypy/branch/out-of-line-guards/pypy/translator/sandbox/test/test_sandbox.py pypy/branch/out-of-line-guards/pypy/translator/tool/cbuild.py pypy/branch/out-of-line-guards/pypy/translator/tool/reftracker.py pypy/branch/out-of-line-guards/site-packages/ (props changed) Log: Merge from trunk r79309:HEAD Modified: pypy/branch/out-of-line-guards/ctypes_configure/configure.py ============================================================================== --- pypy/branch/out-of-line-guards/ctypes_configure/configure.py (original) +++ pypy/branch/out-of-line-guards/ctypes_configure/configure.py Thu Dec 9 16:33:59 2010 @@ -559,6 +559,7 @@ C_HEADER = """ #include #include /* for offsetof() */ +#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); Modified: pypy/branch/out-of-line-guards/lib-python/conftest.py ============================================================================== --- pypy/branch/out-of-line-guards/lib-python/conftest.py (original) +++ pypy/branch/out-of-line-guards/lib-python/conftest.py Thu Dec 9 16:33:59 2010 @@ -139,7 +139,7 @@ RegrTest('test_augassign.py', core=True), RegrTest('test_base64.py'), RegrTest('test_bastion.py'), - RegrTest('test_binascii.py'), + RegrTest('test_binascii.py', usemodules='binascii'), RegrTest('test_binhex.py'), Modified: pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/msvccompiler.py Thu Dec 9 16:33:59 2010 @@ -25,17 +25,20 @@ from distutils import log from distutils.util import get_platform -import _winreg - -RegOpenKeyEx = _winreg.OpenKeyEx -RegEnumKey = _winreg.EnumKey -RegEnumValue = _winreg.EnumValue -RegError = _winreg.error - -HKEYS = (_winreg.HKEY_USERS, - _winreg.HKEY_CURRENT_USER, - _winreg.HKEY_LOCAL_MACHINE, - _winreg.HKEY_CLASSES_ROOT) +try: + import _winreg +except ImportError: + pass +else: + RegOpenKeyEx = _winreg.OpenKeyEx + RegEnumKey = _winreg.EnumKey + RegEnumValue = _winreg.EnumValue + RegError = _winreg.error + + HKEYS = (_winreg.HKEY_USERS, + _winreg.HKEY_CURRENT_USER, + _winreg.HKEY_LOCAL_MACHINE, + _winreg.HKEY_CLASSES_ROOT) VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" VSEXPRESS_BASE = r"Software\Microsoft\VCExpress\%0.1f" Modified: pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py ============================================================================== --- pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py (original) +++ pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py Thu Dec 9 16:33:59 2010 @@ -11,11 +11,15 @@ else: self.fail("could not find a suitable manifest") +class MsvcCompilerSimplerTestCase(unittest.TestCase): + def test_import_module(self): + from distutils.msvccompiler import MSVCCompiler + def test_suite(): if sys.platform == 'win32': return unittest.makeSuite(MsvcCompilerTestCase) else: - return unittest.TestSuite([]) + return unittest.makeSuite(MsvcCompilerSimplerTestCase) if __name__ == "__main__": unittest.main(defaultTest="test_suite") Modified: pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/unixccompiler.py ============================================================================== --- pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/unixccompiler.py (original) +++ pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/distutils/unixccompiler.py Thu Dec 9 16:33:59 2010 @@ -121,7 +121,22 @@ } if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() executables['ranlib'] = ["ranlib"] + executables['linker_so'] += ['-undefined', 'dynamic_lookup'] + + for k, v in executables.iteritems(): + if v and v[0] == 'cc': + v += ['-arch', arch] + # Needed for the filename generation methods provided by the base # class, CCompiler. NB. whoever instantiates/uses a particular Modified: pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/output/test_cProfile ============================================================================== --- pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/output/test_cProfile (original) +++ pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/output/test_cProfile Thu Dec 9 16:33:59 2010 @@ -14,66 +14,66 @@ 4 0.116 0.029 0.120 0.030 test_cProfile.py:78(helper1) 2 0.000 0.000 0.140 0.070 test_cProfile.py:89(helper2_indirect) 8 0.312 0.039 0.400 0.050 test_cProfile.py:93(helper2) - 4 0.000 0.000 0.000 0.000 {append} - 1 0.000 0.000 0.000 0.000 {disable} 12 0.000 0.000 0.012 0.001 {hasattr} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} + 1 0.000 0.000 0.000 0.000 {method 'disable' of 'Profile' objects} 8 0.000 0.000 0.000 0.000 {range} 4 0.000 0.000 0.000 0.000 {sys.exc_info} Ordered by: standard name -Function called... - ncalls tottime cumtime -:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) -test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) - 8 0.000 0.000 {range} -test_cProfile.py:115(__getattr__) -> -test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) - 2 0.040 0.600 test_cProfile.py:60(helper) -test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 20 0.020 0.020 test_cProfile.py:53(mul) -test_cProfile.py:53(mul) -> -test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) - 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) - 6 0.234 0.300 test_cProfile.py:93(helper2) -test_cProfile.py:78(helper1) -> 4 0.000 0.000 {append} - 4 0.000 0.004 {hasattr} - 4 0.000 0.000 {sys.exc_info} -test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) - 2 0.078 0.100 test_cProfile.py:93(helper2) -test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) - 8 0.000 0.008 {hasattr} -{append} -> -{disable} -> -{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) -{range} -> -{sys.exc_info} -> +Function called... + ncalls tottime cumtime +:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) +test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) + 8 0.000 0.000 {range} +test_cProfile.py:115(__getattr__) -> +test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) + 2 0.040 0.600 test_cProfile.py:60(helper) +test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 20 0.020 0.020 test_cProfile.py:53(mul) +test_cProfile.py:53(mul) -> +test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) + 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) + 6 0.234 0.300 test_cProfile.py:93(helper2) +test_cProfile.py:78(helper1) -> 4 0.000 0.004 {hasattr} + 4 0.000 0.000 {method 'append' of 'list' objects} + 4 0.000 0.000 {sys.exc_info} +test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) + 2 0.078 0.100 test_cProfile.py:93(helper2) +test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) + 8 0.000 0.008 {hasattr} +{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) +{method 'append' of 'list' objects} -> +{method 'disable' of 'Profile' objects} -> +{range} -> +{sys.exc_info} -> Ordered by: standard name -Function was called by... - ncalls tottime cumtime -:1() <- -test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) -test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) - 12 0.012 0.012 {hasattr} -test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() -test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) - 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) -test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) -test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) -test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) -test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) -test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) - 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) -{append} <- 4 0.000 0.000 test_cProfile.py:78(helper1) -{disable} <- -{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) - 8 0.000 0.008 test_cProfile.py:93(helper2) -{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) -{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +Function was called by... + ncalls tottime cumtime +:1() <- +test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) +test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) + 12 0.012 0.012 {hasattr} +test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() +test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) + 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) +test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) +test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) +test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) +test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) +test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) + 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) +{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) + 8 0.000 0.008 test_cProfile.py:93(helper2) +{method 'append' of 'list' objects} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +{method 'disable' of 'Profile' objects} <- +{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) +{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) Modified: pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/test_genexps.py ============================================================================== --- pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/test_genexps.py (original) +++ pypy/branch/out-of-line-guards/lib-python/modified-2.5.2/test/test_genexps.py Thu Dec 9 16:33:59 2010 @@ -77,10 +77,10 @@ Verify that parenthesis are required in a statement >>> def f(n): - ... return i*i for i in xrange(n) + ... return i*i for i in xrange(n) #doctest: +ELLIPSIS Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax... Verify that parenthesis are required when used as a keyword argument value Modified: pypy/branch/out-of-line-guards/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_ctypes/array.py Thu Dec 9 16:33:59 2010 @@ -99,6 +99,9 @@ if len(value) > self._length_: raise ValueError("Invalid length") value = self(*value) + elif not isinstance(value, self): + raise TypeError("expected string or Unicode object, %s found" + % (value.__class__.__name__,)) else: if isinstance(value, tuple): if len(value) > self._length_: @@ -107,22 +110,43 @@ return _CDataMeta.from_param(self, value) def array_get_slice_params(self, index): - if index.step is not None: - raise TypeError("3 arg slices not supported (for no reason)") - start = index.start or 0 - stop = index.stop or self._length_ - return start, stop + if hasattr(self, '_length_'): + start, stop, step = index.indices(self._length_) + else: + step = index.step + if step is None: + step = 1 + start = index.start + stop = index.stop + if start is None: + if step > 0: + start = 0 + else: + raise ValueError("slice start is required for step < 0") + if stop is None: + raise ValueError("slice stop is required") + + return start, stop, step def array_slice_setitem(self, index, value): - start, stop = self._get_slice_params(index) - if stop - start != len(value): + start, stop, step = self._get_slice_params(index) + + if ((step < 0 and stop >= start) or + (step > 0 and start >= stop)): + slicelength = 0 + elif step < 0: + slicelength = (stop - start + 1) / step + 1 + else: + slicelength = (stop - start - 1) / step + 1; + + if slicelength != len(value): raise ValueError("Can only assign slices of the same length") - for i in range(start, stop): - self[i] = value[i - start] + for i, j in enumerate(range(start, stop, step)): + self[j] = value[i] def array_slice_getitem(self, index): - start, stop = self._get_slice_params(index) - l = [self[i] for i in range(start, stop)] + start, stop, step = self._get_slice_params(index) + l = [self[i] for i in range(start, stop, step)] letter = getattr(self._type_, '_type_', None) if letter == 'c': return "".join(l) @@ -135,7 +159,8 @@ _ffiargshape = 'P' def __init__(self, *args): - self._buffer = self._ffiarray(self._length_, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(self._length_, autofree=True) for i, arg in enumerate(args): self[i] = arg @@ -162,9 +187,10 @@ self._slice_setitem(index, value) return index = self._fix_index(index) - if ensure_objects(value) is not None: - store_reference(self, index, value._objects) - arg = self._type_._CData_value(value) + cobj = self._type_.from_param(value) + if ensure_objects(cobj) is not None: + store_reference(self, index, cobj._objects) + arg = cobj._get_buffer_value() if self._type_._fficompositesize is None: self._buffer[index] = arg # something more sophisticated, cannot set field directly @@ -183,7 +209,7 @@ return self._length_ def _get_buffer_for_param(self): - return CArgObject(self._buffer.byptr()) + return CArgObject(self, self._buffer.byptr()) def _get_buffer_value(self): return self._buffer.buffer Modified: pypy/branch/out-of-line-guards/lib_pypy/_ctypes/basics.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_ctypes/basics.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_ctypes/basics.py Thu Dec 9 16:33:59 2010 @@ -46,21 +46,6 @@ else: return self.from_param(as_parameter) - def _CData_input(self, value): - """Used when data enters into ctypes from user code. 'value' is - some user-specified Python object, which is converted into a _rawffi - array of length 1 containing the same value according to the - type 'self'. - """ - cobj = self.from_param(value) - return cobj, cobj._get_buffer_for_param() - - def _CData_value(self, value): - cobj = self.from_param(value) - # we don't care here if this stuff will live afterwards, as we're - # interested only in value anyway - return cobj._get_buffer_value() - def _CData_output(self, resbuffer, base=None, index=-1): #assert isinstance(resbuffer, _rawffi.ArrayInstance) """Used when data exits ctypes and goes into user code. @@ -92,13 +77,23 @@ """ simple wrapper around buffer, just for the case of freeing it afterwards """ - def __init__(self, buffer): + def __init__(self, obj, buffer): + self._obj = obj self._buffer = buffer def __del__(self): self._buffer.free() self._buffer = None + def __repr__(self): + return '' % (self._obj,) + + def __eq__(self, other): + return self._obj == other + + def __ne__(self, other): + return self._obj != other + class _CData(object): """ The most basic object for all ctypes types """ @@ -128,7 +123,7 @@ return buffer(self._buffer) def _get_b_base(self): - return self._objects + return self._base _b_base_ = property(_get_b_base) _b_needsfree_ = False Modified: pypy/branch/out-of-line-guards/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_ctypes/builtin.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_ctypes/builtin.py Thu Dec 9 16:33:59 2010 @@ -11,7 +11,8 @@ def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_void_p._CData_value(addr) + cobj = ctypes.c_void_p.from_param(addr) + arg = cobj._get_buffer_value() return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -22,7 +23,8 @@ def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_void_p._CData_value(addr) + cobj = ctypes.c_void_p.from_param(addr) + arg = cobj._get_buffer_value() # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/branch/out-of-line-guards/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_ctypes/function.py Thu Dec 9 16:33:59 2010 @@ -35,6 +35,7 @@ _argtypes_ = None _restype_ = None + _errcheck_ = None _flags_ = 0 _ffiargshape = 'P' _ffishape = 'P' @@ -53,7 +54,15 @@ return self._argtypes_ def _setargtypes(self, argtypes): self._ptr = None - self._argtypes_ = argtypes + if argtypes is None: + self._argtypes_ = None + else: + for i, argtype in enumerate(argtypes): + if not hasattr(argtype, 'from_param'): + raise TypeError( + "item %d in _argtypes_ has no from_param method" % ( + i + 1,)) + self._argtypes_ = argtypes argtypes = property(_getargtypes, _setargtypes) def _getrestype(self): @@ -72,9 +81,24 @@ del self._restype_ restype = property(_getrestype, _setrestype, _delrestype) + def _geterrcheck(self): + return getattr(self, '_errcheck_', None) + def _seterrcheck(self, errcheck): + if not callable(errcheck): + raise TypeError("The errcheck attribute must be callable") + self._errcheck_ = errcheck + def _delerrcheck(self): + try: + del self._errcheck_ + except AttributeError: + pass + errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck) + def _ffishapes(self, args, restype): argtypes = [arg._ffiargshape for arg in args] if restype is not None: + if not isinstance(restype, _CDataMeta): + raise TypeError("invalid result type for callback function") restype = restype._ffiargshape else: restype = 'O' # void @@ -140,6 +164,7 @@ def __call__(self, *args): if self.callable is not None: + args = args[:len(self._argtypes_)] try: res = self.callable(*args) except: @@ -162,13 +187,29 @@ thisarg = None if argtypes is None: - argtypes = self._guess_argtypes(args) - argtypes, argsandobjs = self._wrap_args(argtypes, args) + argtypes = [] + args = self._convert_args(argtypes, args) + argtypes = [type(arg) for arg in args] restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype, thisarg) - resbuffer = funcptr(*[arg._buffer for _, arg in argsandobjs]) - return self._build_result(restype, resbuffer, argtypes, argsandobjs) + resbuffer = funcptr(*[arg._get_buffer_for_param()._buffer + for arg in args]) + result = self._build_result(restype, resbuffer, argtypes, args) + + # The 'errcheck' protocol + if self._errcheck_: + v = self._errcheck_(result, self, args) + # If the errcheck funtion failed, let it throw + # If the errcheck function returned callargs unchanged, + # continue normal processing. + # If the errcheck function returned something else, + # use that as result. + if v is not args: + result = v + + return result + def _getfuncptr(self, argtypes, restype, thisarg=None): if self._ptr is not None and argtypes is self._argtypes_: @@ -212,31 +253,33 @@ raise @staticmethod - def _guess_argtypes(args): + def _conv_param(argtype, arg, index): from ctypes import c_char_p, c_wchar_p, c_void_p, c_int - res = [] - for arg in args: - if hasattr(arg, '_as_parameter_'): - arg = arg._as_parameter_ - if isinstance(arg, str): - res.append(c_char_p) - elif isinstance(arg, unicode): - res.append(c_wchar_p) - elif isinstance(arg, _CData): - res.append(type(arg)) - elif arg is None: - res.append(c_void_p) - #elif arg == 0: - # res.append(c_void_p) - elif isinstance(arg, (int, long)): - res.append(c_int) - else: - raise TypeError("Don't know how to handle %s" % (arg,)) - return res + if argtype is not None: + arg = argtype.from_param(arg) + if hasattr(arg, '_as_parameter_'): + arg = arg._as_parameter_ + + if isinstance(arg, _CData): + # The usual case when argtype is defined + cobj = arg + elif isinstance(arg, str): + cobj = c_char_p(arg) + elif isinstance(arg, unicode): + cobj = c_wchar_p(arg) + elif arg is None: + cobj = c_void_p() + elif isinstance(arg, (int, long)): + cobj = c_int(arg) + else: + raise TypeError("Don't know how to handle %s" % (arg,)) - def _wrap_args(self, argtypes, args): + return cobj + + def _convert_args(self, argtypes, args): wrapped_args = [] consumed = 0 + for i, argtype in enumerate(argtypes): defaultvalue = None if i > 0 and self._paramflags is not None: @@ -263,7 +306,7 @@ val = defaultvalue if val is None: val = 0 - wrapped = argtype._CData_input(val) + wrapped = self._conv_param(argtype, val, consumed) wrapped_args.append(wrapped) continue else: @@ -278,23 +321,22 @@ raise TypeError("Not enough arguments") try: - wrapped = argtype._CData_input(arg) - except (UnicodeError, TypeError), e: + wrapped = self._conv_param(argtype, arg, consumed) + except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) wrapped_args.append(wrapped) consumed += 1 if len(wrapped_args) < len(args): extra = args[len(wrapped_args):] - extra_types = self._guess_argtypes(extra) - for arg, argtype in zip(extra, extra_types): + argtypes = list(argtypes) + for i, arg in enumerate(extra): try: - wrapped = argtype._CData_input(arg) - except (UnicodeError, TypeError), e: + wrapped = self._conv_param(None, arg, i) + except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) wrapped_args.append(wrapped) - argtypes = list(argtypes) + extra_types - return argtypes, wrapped_args + return wrapped_args def _build_result(self, restype, resbuffer, argtypes, argsandobjs): """Build the function result: @@ -307,7 +349,7 @@ if self._com_index: if resbuffer[0] & 0x80000000: raise get_com_error(resbuffer[0], - self._com_iid, argsandobjs[0][0]) + self._com_iid, argsandobjs[0]) else: retval = int(resbuffer[0]) elif restype is not None: @@ -326,8 +368,8 @@ results = [] if self._paramflags: - for argtype, (obj, _), paramflag in zip(argtypes[1:], argsandobjs[1:], - self._paramflags): + for argtype, obj, paramflag in zip(argtypes[1:], argsandobjs[1:], + self._paramflags): if len(paramflag) == 2: idlflag, name = paramflag elif len(paramflag) == 3: Modified: pypy/branch/out-of-line-guards/lib_pypy/_ctypes/pointer.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_ctypes/pointer.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_ctypes/pointer.py Thu Dec 9 16:33:59 2010 @@ -1,7 +1,8 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, cdata_from_address -from _ctypes.basics import sizeof, byref, keepalive_key +from _ctypes.basics import keepalive_key, store_reference, ensure_objects +from _ctypes.basics import sizeof, byref from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem @@ -55,7 +56,8 @@ def set_type(self, TP): ffiarray = _rawffi.Array('P') def __init__(self, value=None): - self._buffer = ffiarray(1, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = ffiarray(1, autofree=True) if value is not None: self.contents = value self._ffiarray = ffiarray @@ -99,7 +101,10 @@ return self._type_._CData_output(self._subarray(index), self, index) def __setitem__(self, index, value): - self._subarray(index)[0] = self._type_._CData_value(value) + cobj = self._type_.from_param(value) + if ensure_objects(cobj) is not None: + store_reference(self, index, cobj._objects) + self._subarray(index)[0] = cobj._get_buffer_value() def __nonzero__(self): return self._buffer[0] != 0 @@ -110,29 +115,32 @@ if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): raise TypeError("cast() argument 2 must be a pointer type, not %s" % (tp,)) - if isinstance(obj, Array): - ptr = tp.__new__(tp) - ptr._buffer = tp._ffiarray(1, autofree=True) - ptr._buffer[0] = obj._buffer - return ptr if isinstance(obj, (int, long)): result = tp() result._buffer[0] = obj return result - if obj is None: + elif obj is None: result = tp() return result - if not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): + elif isinstance(obj, Array): + ptr = tp.__new__(tp) + ptr._buffer = tp._ffiarray(1, autofree=True) + ptr._buffer[0] = obj._buffer + result = ptr + elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) - result = tp() + else: + result = tp() + result._buffer[0] = obj._buffer[0] # The casted objects '_objects' member: - # It must certainly contain the source objects one. + # From now on, both objects will use the same dictionary + # It must certainly contain the source objects # It must contain the source object itself. if obj._ensure_objects() is not None: - result._objects = {keepalive_key(0): obj._objects, - keepalive_key(1): obj} + result._objects = obj._objects + if isinstance(obj._objects, dict): + result._objects[id(obj)] = obj - result._buffer[0] = obj._buffer[0] return result Modified: pypy/branch/out-of-line-guards/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_ctypes/primitive.py Thu Dec 9 16:33:59 2010 @@ -132,8 +132,8 @@ ConvMode.errors) #self._objects = value array = _rawffi.Array('c')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -155,8 +155,8 @@ ConvMode.errors) #self._objects = value array = _rawffi.Array('u')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -174,8 +174,8 @@ def _setvalue(self, value): if isinstance(value, str): array = _rawffi.Array('c')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -271,7 +271,9 @@ def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) - return output.value + if self.__bases__[0] is _SimpleCData: + return output.value + return output def _sizeofinstances(self): return _rawffi.sizeof(self._type_) @@ -287,7 +289,8 @@ _type_ = 'i' def __init__(self, value=DEFAULT_VALUE): - self._buffer = self._ffiarray(1, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(1, autofree=True) if value is not DEFAULT_VALUE: self.value = value @@ -312,7 +315,11 @@ return self.value def __repr__(self): - return "%s(%r)" % (type(self).__name__, self.value) + if type(self).__bases__[0] is _SimpleCData: + return "%s(%r)" % (type(self).__name__, self.value) + else: + return "<%s object at 0x%x>" % (type(self).__name__, + id(self)) def __nonzero__(self): return self._buffer[0] not in (0, '\x00') Modified: pypy/branch/out-of-line-guards/lib_pypy/_ctypes/structure.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_ctypes/structure.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_ctypes/structure.py Thu Dec 9 16:33:59 2010 @@ -34,9 +34,11 @@ if not isinstance(tp, _CDataMeta): raise TypeError("Expected CData subclass, got %s" % (tp,)) import ctypes - all_fields = _fields_[:] - for cls in inspect.getmro(superclass): - all_fields += getattr(cls, '_fields_', []) + all_fields = [] + for cls in reversed(inspect.getmro(superclass)): + # The first field comes from the most base class + all_fields.extend(getattr(cls, '_fields_', [])) + all_fields.extend(_fields_) names = [name for name, ctype in all_fields] rawfields = [(name, ctype._ffishape) for name, ctype in all_fields] @@ -168,7 +170,7 @@ def __init__(self, *args, **kwds): if len(args) > len(self._names): - raise TypeError("too many arguments") + raise TypeError("too many initializers") for name, arg in zip(self._names, args): if name in kwds: raise TypeError("duplicate value for argument %r" % ( Modified: pypy/branch/out-of-line-guards/lib_pypy/_hashlib.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_hashlib.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_hashlib.py Thu Dec 9 16:33:59 2010 @@ -148,21 +148,29 @@ return hash(ctx, name) # shortcut functions +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + at builtinify def openssl_md5(string=''): return new('md5', string) + at builtinify def openssl_sha1(string=''): return new('sha1', string) + at builtinify def openssl_sha224(string=''): return new('sha224', string) + at builtinify def openssl_sha256(string=''): return new('sha256', string) + at builtinify def openssl_sha384(string=''): return new('sha384', string) + at builtinify def openssl_sha512(string=''): return new('sha512', string) - Modified: pypy/branch/out-of-line-guards/lib_pypy/_locale.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_locale.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_locale.py Thu Dec 9 16:33:59 2010 @@ -11,6 +11,9 @@ # load the platform-specific cache made by running locale.ctc.py from ctypes_config_cache._locale_cache import * +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + # Ubuntu Gusty i386 structure class lconv(Structure): @@ -158,6 +161,7 @@ ul = ''.join(ul) string.letters = ul + at builtinify def setlocale(category, locale=None): "(integer,string=None) -> string. Activates/queries locale processing." if locale: @@ -182,6 +186,7 @@ groups.append(0) return groups + at builtinify def localeconv(): "() -> dict. Returns numeric and monetary locale-specific parameters." @@ -215,6 +220,7 @@ } return result + at builtinify def strcoll(s1, s2): "string,string -> int. Compares two strings according to the locale." @@ -233,6 +239,7 @@ # Collate the strings. return _wcscoll(s1, s2) + at builtinify def strxfrm(s): "string -> string. Returns a string that behaves for cmp locale-aware." @@ -246,6 +253,7 @@ _strxfrm(buf, s, n2) return buf.value + at builtinify def getdefaultlocale(): # TODO: Port code from CPython for Windows and Mac OS raise NotImplementedError() @@ -267,26 +275,31 @@ raise ValueError("unsupported langinfo constant") if HAS_LIBINTL: + @builtinify def gettext(msg): """gettext(msg) -> string Return translation of msg.""" return _gettext(msg) + @builtinify def dgettext(domain, msg): """dgettext(domain, msg) -> string Return translation of msg in domain.""" return _dgettext(domain, msg) + @builtinify def dcgettext(domain, msg, category): """dcgettext(domain, msg, category) -> string Return translation of msg in domain and category.""" return _dcgettext(domain, msg, category) + @builtinify def textdomain(domain): """textdomain(domain) -> string Set the C library's textdomain to domain, returning the new domain.""" return _textdomain(domain) + @builtinify def bindtextdomain(domain, dir): """bindtextdomain(domain, dir) -> string Bind the C library's domain to dir.""" @@ -297,6 +310,7 @@ return dirname if HAS_BIND_TEXTDOMAIN_CODESET: + @builtinify def bind_textdomain_codeset(domain, codeset): """bind_textdomain_codeset(domain, codeset) -> string Bind the C library's domain to codeset.""" Modified: pypy/branch/out-of-line-guards/lib_pypy/_marshal.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_marshal.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_marshal.py Thu Dec 9 16:33:59 2010 @@ -6,6 +6,10 @@ import types from _codecs import utf_8_decode, utf_8_encode +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + TYPE_NULL = '0' TYPE_NONE = 'N' TYPE_FALSE = 'F' @@ -645,15 +649,18 @@ version = 1 + at builtinify def dump(x, f, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format m = _Marshaller(f.write) m.dump(x) + at builtinify def load(f): um = _Unmarshaller(f.read) return um.load() + at builtinify def dumps(x, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format buffer = [] @@ -661,6 +668,7 @@ m.dump(x) return ''.join(buffer) + at builtinify def loads(s): um = _FastUnmarshaller(s) return um.load() Modified: pypy/branch/out-of-line-guards/lib_pypy/_minimal_curses.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_minimal_curses.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_minimal_curses.py Thu Dec 9 16:33:59 2010 @@ -35,18 +35,24 @@ # ____________________________________________________________ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + at builtinify def setupterm(termstr, fd): err = ctypes.c_int(0) result = clib.setupterm(termstr, fd, ctypes.byref(err)) if result == ERR: raise error("setupterm() failed (err=%d)" % err.value) + at builtinify def tigetstr(cap): result = clib.tigetstr(cap) if ctypes.cast(result, ctypes.c_void_p).value == ERR: return None return ctypes.cast(result, ctypes.c_char_p).value + at builtinify def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) if result is None: Modified: pypy/branch/out-of-line-guards/lib_pypy/_pypy_interact.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/_pypy_interact.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/_pypy_interact.py Thu Dec 9 16:33:59 2010 @@ -4,6 +4,13 @@ def interactive_console(mainmodule=None): + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, 'ps1'): + sys.ps1 = '>>>> ' + if not hasattr(sys, 'ps2'): + sys.ps2 = '.... ' + # try: from _pypy_irc_topic import some_topic text = "And now for something completely different: ``%s''" % ( @@ -15,6 +22,7 @@ print text except ImportError: pass + # try: from pyrepl.simple_interact import check if not check(): Modified: pypy/branch/out-of-line-guards/lib_pypy/binascii.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/binascii.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/binascii.py Thu Dec 9 16:33:59 2010 @@ -1,3 +1,9 @@ +"""A pure Python implementation of binascii. + +Rather slow and buggy in corner cases. +PyPy provides an RPython version too. +""" + class Error(Exception): pass @@ -277,7 +283,7 @@ if (c > '~' or c == '=' or (header and c == '_') or - (c == '.' and linelen == 0 and (inp == len(data) or + (c == '.' and linelen == 0 and (inp+1 == len(data) or data[inp+1] == '\n' or data[inp+1] == '\r')) or (not istext and (c == '\r' or c == '\n')) or Modified: pypy/branch/out-of-line-guards/lib_pypy/cPickle.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/cPickle.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/cPickle.py Thu Dec 9 16:33:59 2010 @@ -5,6 +5,10 @@ from pickle import * from pickle import __doc__, __version__, format_version, compatible_formats +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + BadPickleGet = KeyError UnpickleableError = PicklingError @@ -31,9 +35,11 @@ def getvalue(self): return self.__f and self.__f.getvalue() + at builtinify def dump(obj, file, protocol=None): Pickler(file, protocol).dump(obj) + at builtinify def dumps(obj, protocol=None): file = StringIO() Pickler(file, protocol).dump(obj) Modified: pypy/branch/out-of-line-guards/lib_pypy/cmath.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/cmath.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/cmath.py Thu Dec 9 16:33:59 2010 @@ -7,7 +7,10 @@ import math from math import e, pi - + +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + # constants _one = complex(1., 0.) @@ -24,6 +27,7 @@ return complex(real, imag) + at builtinify def acos(x): """acos(x) @@ -32,6 +36,7 @@ return -(_prodi(log((x+(_i*sqrt((_one-(x*x)))))))) + at builtinify def acosh(x): """acosh(x) @@ -41,6 +46,7 @@ return z+z + at builtinify def asin(x): """asin(x) @@ -52,6 +58,7 @@ return -(_prodi(log((sqrt_1_minus_x_sq+_prodi(x))))) + at builtinify def asinh(x): """asinh(x) @@ -61,6 +68,7 @@ return z+z + at builtinify def atan(x): """atan(x) @@ -69,6 +77,7 @@ return _halfi*log(((_i+x)/(_i-x))) + at builtinify def atanh(x): """atanh(x) @@ -77,6 +86,7 @@ return _half*log((_one+x)/(_one-x)) + at builtinify def cos(x): """cos(x) @@ -88,6 +98,7 @@ return complex(real, imag) + at builtinify def cosh(x): """cosh(x) @@ -99,6 +110,7 @@ return complex(real, imag) + at builtinify def exp(x): """exp(x) @@ -111,6 +123,7 @@ return complex(real, imag) + at builtinify def log(x, base=None): """log(x) @@ -125,6 +138,7 @@ return complex(real, imag) + at builtinify def log10(x): """log10(x) @@ -137,6 +151,7 @@ return complex(real, imag) + at builtinify def sin(x): """sin(x) @@ -148,6 +163,7 @@ return complex(real, imag) + at builtinify def sinh(x): """sinh(x) @@ -159,6 +175,7 @@ return complex(real, imag) + at builtinify def sqrt(x): """sqrt(x) @@ -184,6 +201,7 @@ _sqrt_half = sqrt(_half) + at builtinify def tan(x): """tan(x) @@ -204,6 +222,7 @@ return complex(real, imag) + at builtinify def tanh(x): """tanh(x) Modified: pypy/branch/out-of-line-guards/lib_pypy/ctypes_support.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/ctypes_support.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/ctypes_support.py Thu Dec 9 16:33:59 2010 @@ -25,7 +25,7 @@ def _where_is_errno(): return standard_c_lib.__errno_location() -elif sys.platform == 'darwin': +elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() Modified: pypy/branch/out-of-line-guards/lib_pypy/grp.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/grp.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/grp.py Thu Dec 9 16:33:59 2010 @@ -9,6 +9,10 @@ from ctypes import Structure, c_char_p, c_int, POINTER from ctypes_support import standard_c_lib as libc +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + gid_t = c_int class GroupStruct(Structure): @@ -64,6 +68,7 @@ return Group(res.contents.gr_name, res.contents.gr_passwd, res.contents.gr_gid, mem) + at builtinify def getgrgid(gid): res = libc.getgrgid(gid) if not res: @@ -71,6 +76,7 @@ raise KeyError(gid) return _group_from_gstruct(res) + at builtinify def getgrnam(name): if not isinstance(name, str): raise TypeError("expected string") @@ -79,6 +85,7 @@ raise KeyError(name) return _group_from_gstruct(res) + at builtinify def getgrall(): libc.setgrent() lst = [] Modified: pypy/branch/out-of-line-guards/lib_pypy/hashlib.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/hashlib.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/hashlib.py Thu Dec 9 16:33:59 2010 @@ -50,53 +50,30 @@ 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' """ +import sys try: import _hashlib except ImportError: _hashlib = None -def __get_builtin_constructor(name): - if name in ('SHA1', 'sha1'): - import sha - return sha.new - elif name in ('MD5', 'md5'): - import md5 - return md5.new - elif name in ('SHA256', 'sha256'): - import _sha256 - return _sha256.sha256 - elif name in ('SHA224', 'sha224'): - import _sha256 - return _sha256.sha224 - elif name in ('SHA512', 'sha512'): - import _sha512 - return _sha512.sha512 - elif name in ('SHA384', 'sha384'): - import _sha512 - return _sha512.sha384 - raise ValueError, "unsupported hash type" - def __hash_new(name, string=''): """new(name, string='') - Return a new hashing object using the named algorithm; optionally initialized with a string. """ try: - if _hashlib: - return _hashlib.new(name, string) - except ValueError: - # If the _hashlib module (OpenSSL) doesn't support the named - # hash, try using our builtin implementations. - # This allows for SHA224/256 and SHA384/512 support even though - # the OpenSSL library prior to 0.9.8 doesn't provide them. - pass - - return __get_builtin_constructor(name)(string) + new = __byname[name] + except KeyError: + raise ValueError("unsupported hash type") + return new(string) new = __hash_new -def _setfuncs(): - # use the wrapper of the C implementation +# ____________________________________________________________ + +__byname = {} +def __use_openssl_funcs(): + # use the wrapper of the C implementation sslprefix = 'openssl_' for opensslfuncname, func in vars(_hashlib).items(): if not opensslfuncname.startswith(sslprefix): @@ -106,23 +83,41 @@ # try them all, some may not work due to the OpenSSL # version not supporting that algorithm. func() - # Use the C function directly (very fast) - globals()[funcname] = func + # Use the C function directly (very fast, but with ctypes overhead) + __byname[funcname] = func except ValueError: - try: - # Use the builtin implementation directly (fast) - globals()[funcname] = __get_builtin_constructor(funcname) - except ValueError: - # this one has no builtin implementation, don't define it - pass + pass + +def __use_builtin_funcs(): + # look up the built-in versions (written in Python or RPython), + # and use the fastest one: + # 1. the one in RPython + # 2. the one from openssl (slower due to ctypes calling overhead) + # 3. the one in pure Python + if 'sha1' not in __byname or 'sha' in sys.builtin_module_names: + import sha + __byname['sha1'] = sha.new + if 'md5' not in __byname or 'md5' in sys.builtin_module_names: + import md5 + __byname['md5'] = md5.new + if 'sha256' not in __byname: + import _sha256 + __byname['sha256'] = _sha256.sha256 + if 'sha224' not in __byname: + import _sha256 + __byname['sha224'] = _sha256.sha224 + if 'sha512' not in __byname: + import _sha512 + __byname['sha512'] = _sha512.sha512 + if 'sha384' not in __byname: + import _sha512 + __byname['sha384'] = _sha512.sha384 + +def __export_funcs(): + for key, value in __byname.items(): + globals()[key] = __byname[key.upper()] = value if _hashlib: - _setfuncs() -else: - # lookup the C function to use directly for the named constructors - md5 = __get_builtin_constructor('md5') - sha1 = __get_builtin_constructor('sha1') - sha224 = __get_builtin_constructor('sha224') - sha256 = __get_builtin_constructor('sha256') - sha384 = __get_builtin_constructor('sha384') - sha512 = __get_builtin_constructor('sha512') + __use_openssl_funcs() +__use_builtin_funcs() +__export_funcs() Modified: pypy/branch/out-of-line-guards/lib_pypy/itertools.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/itertools.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/itertools.py Thu Dec 9 16:33:59 2010 @@ -27,6 +27,9 @@ 'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap', 'takewhile', 'tee'] +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + class chain(object): """Make an iterator that returns elements from the first iterable @@ -565,7 +568,8 @@ def __iter__(self): return self - + + at builtinify def tee(iterable, n=2): """Return n independent iterators from a single iterable. Note : once tee() has made a split, the original iterable Modified: pypy/branch/out-of-line-guards/lib_pypy/msvcrt.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/msvcrt.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/msvcrt.py Thu Dec 9 16:33:59 2010 @@ -17,6 +17,10 @@ except AttributeError: # we are not on windows raise ImportError +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int] open_osfhandle.restype = ctypes.c_int @@ -34,6 +38,7 @@ _locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] _locking.restype = ctypes.c_int + at builtinify def locking(fd, mode, nbytes): '''lock or unlock a number of bytes in a file.''' rv = _locking(fd, mode, nbytes) Modified: pypy/branch/out-of-line-guards/lib_pypy/pwd.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/pwd.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/pwd.py Thu Dec 9 16:33:59 2010 @@ -17,6 +17,10 @@ from ctypes_support import standard_c_lib as libc from ctypes import Structure, POINTER, c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + uid_t = c_int gid_t = c_int @@ -79,10 +83,12 @@ _endpwent.argtypes = None _endpwent.restype = None + at builtinify def mkpwent(pw): pw = pw.contents return struct_passwd(pw) + at builtinify def getpwuid(uid): """ getpwuid(uid) -> (pw_name,pw_passwd,pw_uid, @@ -95,6 +101,7 @@ raise KeyError("getpwuid(): uid not found: %s" % uid) return mkpwent(pw) + at builtinify def getpwnam(name): """ getpwnam(name) -> (pw_name,pw_passwd,pw_uid, @@ -109,9 +116,10 @@ raise KeyError("getpwname(): name not found: %s" % name) return mkpwent(pw) + at builtinify def getpwall(): """ - "getpwall() -> list_of_entries + getpwall() -> list_of_entries Return a list of all available password database entries, in arbitrary order. See pwd.__doc__ for more on password database entries. """ Modified: pypy/branch/out-of-line-guards/lib_pypy/pyexpat.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/pyexpat.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/pyexpat.py Thu Dec 9 16:33:59 2010 @@ -7,6 +7,9 @@ # load the platform-specific cache made by running pyexpat.ctc.py from ctypes_config_cache._pyexpat_cache import * +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + lib = ctypes.CDLL(ctypes.util.find_library('expat')) @@ -425,9 +428,11 @@ new_parser._set_unknown_encoding_handler() return new_parser + at builtinify def ErrorString(errno): return XML_ErrorString(errno)[:200] + at builtinify def ParserCreate(encoding=None, namespace_separator=None, intern=None): if (not isinstance(encoding, str) and not encoding is None): Modified: pypy/branch/out-of-line-guards/lib_pypy/pypy_test/test_hashlib.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/pypy_test/test_hashlib.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/pypy_test/test_hashlib.py Thu Dec 9 16:33:59 2010 @@ -2,10 +2,20 @@ from ..ctypes_config_cache import rebuild rebuild.rebuild_one('hashlib.ctc.py') +from . import hack___pypy__ from .. import hashlib, _hashlib def test_unicode(): - assert isinstance(hashlib.new('sha1', u'xxx'), _hashlib.hash) + assert isinstance(hashlib.new('sha256', u'xxx'), _hashlib.hash) + +pure_python_version = { + 'md5': 'md5.new', + 'sha1': 'sha.new', + 'sha224': '_sha256.sha224', + 'sha256': '_sha256.sha256', + 'sha384': '_sha512.sha384', + 'sha512': '_sha512.sha512', + } def test_attributes(): for name, expected_size in {'md5': 16, @@ -30,7 +40,10 @@ assert hexdigest == h.hexdigest() # also test the pure Python implementation - h = hashlib.__get_builtin_constructor(name)('') + modname, constructor = pure_python_version[name].split('.') + mod = __import__('lib_pypy.' + modname, None, None, ['__doc__']) + builder = getattr(mod, constructor) + h = builder('') assert h.digest_size == expected_size assert h.digestsize == expected_size # Modified: pypy/branch/out-of-line-guards/lib_pypy/pypy_test/test_structseq.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/pypy_test/test_structseq.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/pypy_test/test_structseq.py Thu Dec 9 16:33:59 2010 @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from .._structseq import * +from .._structseq import structseqfield, structseqtype class mydata: Modified: pypy/branch/out-of-line-guards/lib_pypy/readline.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/readline.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/readline.py Thu Dec 9 16:33:59 2010 @@ -6,8 +6,4 @@ are only stubs at the moment. """ -# Note that PyPy contains also a built-in module 'readline' which will hide -# this one if compiled in. However the built-in module is incomplete; -# don't use it. - from pyrepl.readline import * Modified: pypy/branch/out-of-line-guards/lib_pypy/resource.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/resource.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/resource.py Thu Dec 9 16:33:59 2010 @@ -11,6 +11,10 @@ from errno import EINVAL, EPERM import _structseq +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + class error(Exception): pass @@ -77,6 +81,7 @@ ru_nvcsw = _structseq.structseqfield(14) ru_nivcsw = _structseq.structseqfield(15) + at builtinify def rlimit_check_bounds(rlim_cur, rlim_max): if rlim_cur > rlim_t_max: raise ValueError("%d does not fit into rlim_t" % rlim_cur) @@ -89,6 +94,7 @@ ("rlim_max", rlim_t), ) + at builtinify def getrusage(who): ru = _struct_rusage() ret = _getrusage(who, byref(ru)) @@ -116,6 +122,7 @@ ru.ru_nivcsw, )) + at builtinify def getrlimit(resource): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -127,6 +134,7 @@ raise error(errno) return (rlim.rlim_cur, rlim.rlim_max) + at builtinify def setrlimit(resource, rlim): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -143,6 +151,7 @@ else: raise error(errno) + at builtinify def getpagesize(): pagesize = 0 if _getpagesize: Modified: pypy/branch/out-of-line-guards/lib_pypy/syslog.py ============================================================================== --- pypy/branch/out-of-line-guards/lib_pypy/syslog.py (original) +++ pypy/branch/out-of-line-guards/lib_pypy/syslog.py Thu Dec 9 16:33:59 2010 @@ -15,6 +15,10 @@ from ctypes_support import standard_c_lib as libc from ctypes import c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + # Real prototype is: # void syslog(int priority, const char *format, ...); # But we also need format ("%s") and one format argument (message) @@ -34,9 +38,11 @@ _setlogmask.argtypes = (c_int,) _setlogmask.restype = c_int + at builtinify def openlog(ident, option, facility): _openlog(ident, option, facility) + at builtinify def syslog(arg1, arg2=None): if arg2 is not None: priority, message = arg1, arg2 @@ -44,15 +50,19 @@ priority, message = LOG_INFO, arg1 _syslog(priority, "%s", message) + at builtinify def closelog(): _closelog() + at builtinify def setlogmask(mask): return _setlogmask(mask) + at builtinify def LOG_MASK(pri): return (1 << pri) + at builtinify def LOG_UPTO(pri): return (1 << (pri + 1)) - 1 Modified: pypy/branch/out-of-line-guards/pypy/annotation/annrpython.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/annrpython.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/annrpython.py Thu Dec 9 16:33:59 2010 @@ -145,7 +145,7 @@ classdef.add_source_for_attribute(attr, classdef.classdesc) self.bookkeeper assert isinstance(s_result, annmodel.SomePBC) - olddesc = s_result.descriptions.iterkeys().next() + olddesc = s_result.any_description() desc = olddesc.bind_self(classdef) args = self.bookkeeper.build_args("simple_call", args_s[:]) desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc], Modified: pypy/branch/out-of-line-guards/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/binaryop.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/binaryop.py Thu Dec 9 16:33:59 2010 @@ -861,7 +861,7 @@ def getitem((p, obj)): assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - def setitem((p, obj)): + def setitem((p, obj), s_value): assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) class __extend__(pairtype(SomeObject, SomePtr)): Modified: pypy/branch/out-of-line-guards/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/bookkeeper.py Thu Dec 9 16:33:59 2010 @@ -262,7 +262,7 @@ args_s, s_result) def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result): - descs = s_callable.descriptions.keys() + descs = list(s_callable.descriptions) if not descs: return family = descs[0].getcallfamily() @@ -590,7 +590,7 @@ assert s_attr.is_constant() attr = s_attr.const - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue @@ -633,7 +633,7 @@ """Analyse a call to a SomePBC() with the given args (list of annotations). """ - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue first = descs[0] Modified: pypy/branch/out-of-line-guards/pypy/annotation/description.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/description.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/description.py Thu Dec 9 16:33:59 2010 @@ -636,6 +636,24 @@ return self return None + def maybe_return_immutable_list(self, attr, s_result): + # hack: 'x.lst' where lst is listed in _immutable_fields_ as 'lst[*]' + # should really return an immutable list as a result. Implemented + # by changing the result's annotation (but not, of course, doing an + # actual copy in the rtyper). Tested in pypy.rpython.test.test_rlist, + # test_immutable_list_out_of_instance. + search = '%s[*]' % (attr,) + cdesc = self + while cdesc is not None: + if '_immutable_fields_' in cdesc.classdict: + if search in cdesc.classdict['_immutable_fields_'].value: + s_result.listdef.never_resize() + s_copy = s_result.listdef.offspring() + s_copy.listdef.mark_as_immutable() + return s_copy + cdesc = cdesc.basedesc + return s_result # common case + def consider_call_site(bookkeeper, family, descs, args, s_result): from pypy.annotation.model import SomeInstance, SomePBC, s_None if len(descs) == 1: @@ -654,7 +672,7 @@ if isinstance(s_init, SomePBC): assert len(s_init.descriptions) == 1, ( "unexpected dynamic __init__?") - initfuncdesc = s_init.descriptions.keys()[0] + initfuncdesc, = s_init.descriptions if isinstance(initfuncdesc, FunctionDesc): initmethdesc = bookkeeper.getmethoddesc(initfuncdesc, classdef, @@ -782,8 +800,8 @@ desc.selfclassdef, desc.name, commonflags) - del descs[desc] - descs[newdesc] = None + descs.remove(desc) + descs.add(newdesc) # --- case 1 --- groups = {} @@ -798,7 +816,7 @@ for desc2 in group: cdef2 = desc2.selfclassdef if cdef1 is not cdef2 and cdef1.issubclass(cdef2): - del descs[desc1] + descs.remove(desc1) break simplify_desc_set = staticmethod(simplify_desc_set) Modified: pypy/branch/out-of-line-guards/pypy/annotation/listdef.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/listdef.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/listdef.py Thu Dec 9 16:33:59 2010 @@ -6,11 +6,16 @@ class TooLateForChange(Exception): pass +class ListChangeUnallowed(Exception): + pass + class ListItem(object): mutated = False # True for lists mutated after creation resized = False # True for lists resized after creation range_step = None # the step -- only for lists only created by a range() dont_change_any_more = False # set to True when too late for changes + immutable = False # for getattr out of _immutable_fields_ = ['attr[*]'] + must_not_resize = False # make_sure_not_resized() # what to do if range_step is different in merge. # - if one is a list (range_step is None), unify to a list. @@ -26,7 +31,6 @@ self.bookkeeper = bookkeeper self.itemof = {} # set of all ListDefs using this ListItem self.read_locations = {} - self.dont_resize = False if bookkeeper is None: self.dont_change_any_more = True @@ -34,12 +38,15 @@ if not self.mutated: if self.dont_change_any_more: raise TooLateForChange + self.immutable = False self.mutated = True def resize(self): if not self.resized: - if self.dont_change_any_more or self.dont_resize: + if self.dont_change_any_more: raise TooLateForChange + if self.must_not_resize: + raise ListChangeUnallowed("resizing list") self.resized = True def setrangestep(self, step): @@ -63,11 +70,13 @@ # things more general self, other = other, self - if other.dont_resize: - if self.resized: - raise TooLateForChange() - self.dont_resize = True - if other.mutated: self.mutate() + self.immutable &= other.immutable + if other.must_not_resize: + if self.resized: + raise ListChangeUnallowed("list merge with a resized") + self.must_not_resize = True + if other.mutated: + self.mutate() if other.resized: self.resize() if other.range_step != self.range_step: @@ -176,9 +185,11 @@ self.listitem.generalize(s_value) def __repr__(self): - return '<[%r]%s%s>' % (self.listitem.s_value, + return '<[%r]%s%s%s%s>' % (self.listitem.s_value, self.listitem.mutated and 'm' or '', - self.listitem.resized and 'r' or '') + self.listitem.resized and 'r' or '', + self.listitem.immutable and 'I' or '', + self.listitem.must_not_resize and '!R' or '') def mutate(self): self.listitem.mutate() @@ -189,13 +200,20 @@ def never_resize(self): if self.listitem.resized: - raise TooLateForChange() - self.listitem.dont_resize = True + raise ListChangeUnallowed("list already resized") + self.listitem.must_not_resize = True - def never_mutate(self): - if self.listitem.resized or self.listitem.mutated: - raise TooLateForChange() - self.listitem.dont_change_any_more = True + def mark_as_immutable(self): + # Sets the 'immutable' flag. Note that unlike "never resized", + # the immutable flag is only a hint. It is cleared again e.g. + # when we merge with a "normal" list that doesn't have it. It + # is thus expected to live only shortly, mostly for the case + # of writing 'x.list[n]'. + self.never_resize() + if not self.listitem.mutated: + self.listitem.immutable = True + #else: it's fine, don't set immutable=True at all (see + # test_can_merge_immutable_list_with_regular_list) MOST_GENERAL_LISTDEF = ListDef(None, SomeObject()) Modified: pypy/branch/out-of-line-guards/pypy/annotation/model.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/model.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/model.py Thu Dec 9 16:33:59 2010 @@ -356,8 +356,8 @@ immutable = True def __init__(self, descriptions, can_be_None=False, subset_of=None): - # descriptions is a set of Desc instances. - descriptions = dict.fromkeys(descriptions) + # descriptions is a set of Desc instances + descriptions = set(descriptions) self.descriptions = descriptions self.can_be_None = can_be_None self.subset_of = subset_of @@ -379,6 +379,9 @@ if desc.pyobj is not None: self.const = desc.pyobj + def any_description(self): + return iter(self.descriptions).next() + def getKind(self): "Return the common Desc class of all descriptions in this PBC." kinds = {} Modified: pypy/branch/out-of-line-guards/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/specialize.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/specialize.py Thu Dec 9 16:33:59 2010 @@ -345,7 +345,8 @@ key.append(s.const) elif isinstance(s, SomePBC) and len(s.descriptions) == 1: # for test_specialize_arg_bound_method - key.append(s.descriptions.keys()[0]) + desc, = s.descriptions + key.append(desc) else: raise Exception("specialize:arg(%d): argument not constant: %r" % (i, s)) Modified: pypy/branch/out-of-line-guards/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/test/test_annrpython.py Thu Dec 9 16:33:59 2010 @@ -10,7 +10,7 @@ from pypy.translator.translator import graphof as tgraphof from pypy.annotation import policy from pypy.annotation import specialize -from pypy.annotation.listdef import ListDef, TooLateForChange +from pypy.annotation.listdef import ListDef, ListChangeUnallowed from pypy.annotation.dictdef import DictDef from pypy.objspace.flow.model import * from pypy.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong @@ -1010,7 +1010,7 @@ bookkeeper = a.bookkeeper def getmdesc(bmeth): - return bookkeeper.immutablevalue(bmeth).descriptions.keys()[0] + return bookkeeper.immutablevalue(bmeth).any_description() mdescA_m = getmdesc(A().m) mdescC_m = getmdesc(C().m) @@ -2862,7 +2862,7 @@ assert s.items[0].flags == {'access_directly': True} assert isinstance(s.items[1], annmodel.SomePBC) assert len(s.items[1].descriptions) == 1 - assert s.items[1].descriptions.keys()[0].flags == {'access_directly': + assert s.items[1].any_description().flags == {'access_directly': True} assert isinstance(s.items[2], annmodel.SomeInstance) assert s.items[2].flags == {'access_directly': True} @@ -3206,7 +3206,7 @@ l.append(4) a = self.RPythonAnnotator() - py.test.raises(TooLateForChange, a.build_types, g, []) + py.test.raises(ListChangeUnallowed, a.build_types, g, []) assert called def test_listitem_no_mutating2(self): @@ -3229,7 +3229,7 @@ a = self.RPythonAnnotator() a.translator.config.translation.list_comprehension_operations = True - py.test.raises(TooLateForChange, a.build_types, fn, [int]) + py.test.raises(ListChangeUnallowed, a.build_types, fn, [int]) def test_listitem_never_resize(self): from pypy.rlib.debug import check_annotation @@ -3243,7 +3243,7 @@ check_annotation(l, checker) a = self.RPythonAnnotator() - py.test.raises(TooLateForChange, a.build_types, f, []) + py.test.raises(ListChangeUnallowed, a.build_types, f, []) def test_len_of_empty_list(self): @@ -3357,6 +3357,87 @@ # not a constant: both __enter__ and __exit__ have been annotated assert not s.is_constant() + def test_make_sure_not_resized(self): + from pypy.rlib.debug import make_sure_not_resized + + def pycode(consts): + make_sure_not_resized(consts) + def build1(): + return pycode(consts=[1]) + def build2(): + return pycode(consts=[0]) + def fn(): + build1() + build2() + + a = self.RPythonAnnotator() + a.translator.config.translation.list_comprehension_operations = True + a.build_types(fn, []) + # assert did not raise ListChangeUnallowed + + def test_return_immutable_list(self): + class A: + _immutable_fields_ = 'lst[*]' + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + return a.lst + + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert s.listdef.listitem.immutable + + def test_immutable_list_is_actually_resized(self): + class A: + _immutable_fields_ = 'lst[*]' + def f(n): + a = A() + l1 = [n] + l1.append(n+1) + a.lst = l1 + return a.lst + + a = self.RPythonAnnotator() + py.test.raises(ListChangeUnallowed, a.build_types, f, [int]) + + def test_can_merge_immutable_list_with_regular_list(self): + class A: + _immutable_fields_ = 'lst[*]' + def foo(lst): + pass + + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + if n > 0: + foo(a.lst) + else: + lst = [0] + lst[0] = n + foo(lst) + + a = self.RPythonAnnotator() + a.build_types(f, [int]) + + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + if n > 0: + lst = [0] + lst[0] = n + foo(lst) + else: + foo(a.lst) + + a = self.RPythonAnnotator() + a.build_types(f, [int]) + def g(n): return [0,1,2,n] Modified: pypy/branch/out-of-line-guards/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/annotation/unaryop.py (original) +++ pypy/branch/out-of-line-guards/pypy/annotation/unaryop.py Thu Dec 9 16:33:59 2010 @@ -615,6 +615,9 @@ if basedef.classdesc.all_enforced_attrs is not None: if attr in basedef.classdesc.all_enforced_attrs: raise HarmlesslyBlocked("get enforced attr") + elif isinstance(s_result, SomeList): + s_result = ins.classdef.classdesc.maybe_return_immutable_list( + attr, s_result) return s_result return SomeObject() getattr.can_only_throw = [] Modified: pypy/branch/out-of-line-guards/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/config/pypyoption.py (original) +++ pypy/branch/out-of-line-guards/pypy/config/pypyoption.py Thu Dec 9 16:33:59 2010 @@ -31,7 +31,7 @@ "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", - "_bisect"] + "_bisect", "binascii"] )) translation_modules = default_modules.copy() @@ -76,6 +76,7 @@ "_rawffi": [("objspace.usemodules.struct", True)], "cpyext": [("translation.secondaryentrypoints", "cpyext"), ("translation.shared", sys.platform == "win32")], + "_ffi": [("translation.jit_ffi", True)], } module_import_dependencies = { @@ -161,7 +162,6 @@ suggests=[("objspace.allworkingmodules", False)]), BoolOption("geninterp", "specify whether geninterp should be used", - cmdline=None, default=True), BoolOption("logbytecodes", Modified: pypy/branch/out-of-line-guards/pypy/config/translationoption.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/config/translationoption.py (original) +++ pypy/branch/out-of-line-guards/pypy/config/translationoption.py Thu Dec 9 16:33:59 2010 @@ -3,6 +3,7 @@ from pypy.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption, FloatOption from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config from pypy.config.config import ConfigError +from pypy.config.support import detect_number_of_processors DEFL_INLINE_THRESHOLD = 32.4 # just enough to inline add__Int_Int() # and just small enough to prevend inlining of some rlist functions. @@ -113,13 +114,10 @@ ChoiceOption("jit_backend", "choose the backend for the JIT", ["auto", "x86", "x86-without-sse2", "llvm"], default="auto", cmdline="--jit-backend"), - ChoiceOption("jit_debug", "the amount of debugging dumps made by the JIT", - ["off", "profile", "steps", "detailed"], - default="profile", # XXX for now - cmdline="--jit-debug"), ChoiceOption("jit_profiler", "integrate profiler support into the JIT", ["off", "oprofile"], default="off"), + BoolOption("jit_ffi", "optimize libffi calls", default=False), # misc BoolOption("verbose", "Print extra information", default=False), @@ -171,7 +169,7 @@ default=False, negation=False), IntOption("make_jobs", "Specify -j argument to make for compilation" " (C backend only)", - cmdline="--make-jobs", default=1), + cmdline="--make-jobs", default=detect_number_of_processors()), # Flags of the TranslationContext: BoolOption("simplifying", "Simplify flow graphs", default=True), Modified: pypy/branch/out-of-line-guards/pypy/conftest.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/conftest.py (original) +++ pypy/branch/out-of-line-guards/pypy/conftest.py Thu Dec 9 16:33:59 2010 @@ -336,13 +336,15 @@ self.runtest_finish() def runtest_open(self): - leakfinder.start_tracking_allocations() + if not getattr(self.obj, 'dont_track_allocations', False): + leakfinder.start_tracking_allocations() def runtest_perform(self): super(PyPyTestFunction, self).runtest() def runtest_close(self): - if leakfinder.TRACK_ALLOCATIONS: + if (not getattr(self.obj, 'dont_track_allocations', False) + and leakfinder.TRACK_ALLOCATIONS): self._pypytest_leaks = leakfinder.stop_tracking_allocations(False) else: # stop_tracking_allocations() already called self._pypytest_leaks = None Modified: pypy/branch/out-of-line-guards/pypy/doc/cpython_differences.txt ============================================================================== --- pypy/branch/out-of-line-guards/pypy/doc/cpython_differences.txt (original) +++ pypy/branch/out-of-line-guards/pypy/doc/cpython_differences.txt Thu Dec 9 16:33:59 2010 @@ -20,16 +20,21 @@ __builtin__ `__pypy__`_ + _ast + _bisect _codecs _lsprof `_minimal_curses`_ _random `_rawffi`_ + _ssl _socket _sre _weakref + array bz2 cStringIO + `cpyext`_ crypt errno exceptions @@ -72,7 +77,7 @@ * Supported by being rewritten in pure Python (possibly using ``ctypes``): see the `lib_pypy/`_ directory. Examples of modules that we - support this way: ``ctypes``, ``array``, ``cPickle``, + support this way: ``ctypes``, ``cPickle``, ``cStringIO``, ``cmath``, ``dbm`` (?), ``datetime``, ``binascii``... Note that some modules are both in there and in the list above; by default, the built-in module is used (but can be disabled @@ -80,11 +85,13 @@ The extension modules (i.e. modules written in C, in the standard CPython) that are neither mentioned above nor in `lib_pypy/`_ are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) .. the nonstandard modules are listed below... .. _`__pypy__`: __pypy__-module.html .. _`_rawffi`: ctypes-implementation.html .. _`_minimal_curses`: config/objspace.usemodules._minimal_curses.html +.. _`cpyext`: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html .. _Stackless: stackless.html @@ -129,12 +136,10 @@ .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html -The built-in function ``id()`` returns numbers that are not addresses -for most of PyPy's garbage collectors. -This is most visible in the default repr: a typical PyPy object can -pretend to be located ``at 0x00000009``. This is just its ``id()``, not -its real address (because an object can move around in some GCs). Calling -``id`` a lot can lead to performance problem. +Using the default GC called ``minimark``, the built-in function ``id()`` +works like it does in CPython. With other GCs it returns numbers that +are not real addresses (because an object can move around several times) +and calling it a lot can lead to performance problem. Note that if you have a long chain of objects, each with a reference to the next one, and each with a __del__, PyPy's GC will perform badly. On Modified: pypy/branch/out-of-line-guards/pypy/doc/faq.txt ============================================================================== --- pypy/branch/out-of-line-guards/pypy/doc/faq.txt (original) +++ pypy/branch/out-of-line-guards/pypy/doc/faq.txt Thu Dec 9 16:33:59 2010 @@ -47,7 +47,7 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has been a part of 1.3 release, but support is still +change) on trunk. It has been a part of 1.4 release, but support is still in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules Modified: pypy/branch/out-of-line-guards/pypy/doc/index.txt ============================================================================== --- pypy/branch/out-of-line-guards/pypy/doc/index.txt (original) +++ pypy/branch/out-of-line-guards/pypy/doc/index.txt Thu Dec 9 16:33:59 2010 @@ -8,7 +8,7 @@ Getting into PyPy ... ============================================= -* `Release 1.3`_: the latest official release +* `Release 1.4`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -56,4 +56,4 @@ .. _`Documentation`: docindex.html .. _`Getting Started`: getting-started.html .. _papers: extradoc.html -.. _`Release 1.3`: http://pypy.org/download.html +.. _`Release 1.4`: http://pypy.org/download.html Modified: pypy/branch/out-of-line-guards/pypy/doc/release-1.4.0.txt ============================================================================== --- pypy/branch/out-of-line-guards/pypy/doc/release-1.4.0.txt (original) +++ pypy/branch/out-of-line-guards/pypy/doc/release-1.4.0.txt Thu Dec 9 16:33:59 2010 @@ -2,27 +2,58 @@ PyPy 1.4: Ouroboros in practice =============================== -Hello. - We're pleased to announce the 1.4 release of PyPy. This is a major breakthrough in our long journey, as PyPy 1.4 is the first PyPy release that can translate -itself faster than CPython. Starting today, we plan to start using PyPy for our -own development. +itself faster than CPython. Starting today, we are using PyPy more for +our every-day development. So may you :) You can download it here: + + http://pypy.org/download.html + +What is PyPy +============ + +PyPy is a very compliant Python interpreter, almost a drop-in replacement +for CPython. It's fast (`pypy 1.4 and cpython 2.6`_ comparison) -Among other features, this release includes numerous performance improvements +Among its new features, this release includes numerous performance improvements (which made fast self-hosting possible), a 64-bit JIT backend, as well as serious stabilization. As of now, we can consider the 32-bit and 64-bit -linux versions of PyPy stable enoughto run in production. +linux versions of PyPy stable enough to run `in production`_. + +Numerous speed achievements are described on `our blog`_. Normalized speed +charts comparing `pypy 1.4 and pypy 1.3`_ as well as `pypy 1.4 and cpython 2.6`_ +are available on benchmark website. For the impatient: yes, we got a lot faster! More highlights =============== -* Virtualenv support: now PyPy is fully compatible with virtualenv_: note that +* PyPy's built-in Just-in-Time compiler is fully transparent and + automatically generated; it now also has very reasonable memory + requirements. The total memory used by a very complex and + long-running process (translating PyPy itself) is within 1.5x to + at most 2x the memory needed by CPython, for a speed-up of 2x. + +* More compact instances. All instances are as compact as if + they had ``__slots__``. This can give programs a big gain in + memory. (In the example of translation above, we already have + carefully placed ``__slots__``, so there is no extra win.) + +* `Virtualenv support`_: now PyPy is fully compatible with virtualenv_: note that to use it, you need a recent version of virtualenv (>= 1.5). * Faster (and JITted) regular expressions - huge boost in speeding up - sre module. + the `re` module. -* Faster (and JITted) calls to functions like map(). +* Other speed improvements, like JITted calls to functions like map(). .. _virtualenv: http://pypi.python.org/pypi/virtualenv +.. _`Virtualenv support`: http://morepypy.blogspot.com/2010/08/using-virtualenv-with-pypy.html +.. _`in production`: http://morepypy.blogspot.com/2010/11/running-large-radio-telescope-software.html +.. _`our blog`: http://morepypy.blogspot.com +.. _`pypy 1.4 and pypy 1.3`: http://speed.pypy.org/comparison/?exe=1%2B41,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=1%2B41&chart=normal+bars +.. _`pypy 1.4 and cpython 2.6`: http://speed.pypy.org/comparison/?exe=2%2B35,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=2%2B35&chart=normal+bars + +Cheers, + +Carl Friedrich Bolz, Antonio Cuni, Maciej Fijalkowski, +Amaury Forgeot d'Arc, Armin Rigo and the PyPy team Modified: pypy/branch/out-of-line-guards/pypy/doc/sprint-reports.txt ============================================================================== --- pypy/branch/out-of-line-guards/pypy/doc/sprint-reports.txt (original) +++ pypy/branch/out-of-line-guards/pypy/doc/sprint-reports.txt Thu Dec 9 16:33:59 2010 @@ -30,6 +30,17 @@ * `D?sseldorf (October 2006)`_ * `Leysin (January 2007)`_ * `Hildesheim (Feb 2007)`_ (also `EU report writing sprint`_) + * `G?teborg (November 2007)`_ + * `Leysin (January 2008)`_ + * `Berlin (May 2008)`_ + * `Vilnius after EuroPython (July 2008)`_ + * `D?sseldorf (August 2008)`_ + * `Wroclaw (February 2009)`_ + * `Leysin (April 2009)`_ + * `G?teborg (August 2009)`_ + * `D?sseldorf (November 2009)`_ + * `CERN (July 2010)`_ + * `D?sseldorf (October 2010)`_ .. _Hildesheim (Feb 2003): http://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html .. _Gothenburg (May 2003): http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt @@ -55,3 +66,15 @@ .. _Hildesheim (Feb 2007): http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/sprint-report.txt .. _`EU report writing sprint`: http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/eu-report-sprint-report.txt .. _`PyCon/Dallas (Feb 2006)`: http://codespeak.net/pypy/extradoc/sprintinfo/pycon06/sprint-report.txt + .. _`G?teborg (November 2007)`: http://morepypy.blogspot.com/2007_11_01_archive.html + .. _`Leysin (January 2008)`: http://morepypy.blogspot.com/2008/01/leysin-winter-sport-sprint-started.html + .. _`Berlin (May 2008)`: http://morepypy.blogspot.com/2008_05_01_archive.html + .. _`Vilnius after EuroPython (July 2008)`: http://morepypy.blogspot.com/2008/07/europython-2008-pypy-talks-and-sprint.html + .. _`D?sseldorf (August 2008)`: http://morepypy.blogspot.com/2008_10_01_archive.html + .. _`Wroclaw (February 2009)`: http://morepypy.blogspot.com/2009/02/wroclaw-2009-sprint-progress-report.html + .. _`Leysin (April 2009)`: http://morepypy.blogspot.com/2009/04/leysin-sprint-report.html + .. _`G?teborg (August 2009)`: http://morepypy.blogspot.com/2009/08/gothenburg-jit-sprint-report.html + .. _`D?sseldorf (November 2009)`: http://morepypy.blogspot.com/2009/11/dusseldorf-sprint-report.html + .. _`CERN (July 2010)`: http://morepypy.blogspot.com/2010/07/cern-sprint-report-wrapping-c-libraries.html + .. _`D?sseldorf (October 2010)`: http://morepypy.blogspot.com/2010/10/dusseldorf-sprint-report-2010.html + Modified: pypy/branch/out-of-line-guards/pypy/doc/statistic/release_dates.dat ============================================================================== --- pypy/branch/out-of-line-guards/pypy/doc/statistic/release_dates.dat (original) +++ pypy/branch/out-of-line-guards/pypy/doc/statistic/release_dates.dat Thu Dec 9 16:33:59 2010 @@ -7,3 +7,7 @@ 2006-06-25,"PyPy 0.9" 2007-02-17,"PyPy 0.99" 2007-03-27,"PyPy 1.0" +2009-04-28,"PyPy 1.1" +2010-03-12,"PyPy 1.2" +2010-06-26,"PyPy 1.3" +2010-11-26,"PyPy 1.4" Modified: pypy/branch/out-of-line-guards/pypy/doc/statistic/sprint_dates.dat ============================================================================== --- pypy/branch/out-of-line-guards/pypy/doc/statistic/sprint_dates.dat (original) +++ pypy/branch/out-of-line-guards/pypy/doc/statistic/sprint_dates.dat Thu Dec 9 16:33:59 2010 @@ -1,26 +1,39 @@ PyPy sprints location, begin, end "Hildesheim",2003-02-17,2003-02-23 -"Gothenburg",2003-05-24,2003-05-31 -"LovainLaNeuve",2003-06-21,2003-06-24 +"G??teborg",2003-05-24,2003-05-31 +"Louvain-la-Neuve",2003-06-21,2003-06-24 "Berlin",2003-09-29,2003-10-04 "Amsterdam",2003-12-14,2003-12-21 -"Europython/Gothenburg",2004-06-01,2004-06-07 +"Europython/G??teborg",2004-06-01,2004-06-07 "Vilnius",2004-11-15,2004-11-23 "Leysin",2005-01-22,2005-01-29 "PyCon/Washington",2005-03-19,2005-03-22 -"Europython/Gothenburg",2005-07-01,2005-07-07 +"Europython/G??teborg",2005-07-01,2005-07-07 "Hildesheim",2005-07-25,2005-07-31 "Heidelberg",2005-08-22,2005-08-29 "Paris",2005-10-10,2005-10-16 -"Gothenburg",2005-12-05,2005-12-11 +"G??teborg",2005-12-05,2005-12-11 "Mallorca",2006-01-23,2006-01-29 "Pycon/Dallas",2006-02-27,2006-03-02 "Louvain-la-Neuve",2006-03-06,2006-03-10 "Japan",2006-04-23,2006-04-29 -"Duesseldorf",2006-06-02,2006-06-09 +"D??sseldorf",2006-06-02,2006-06-09 "Europython/Genf",2006-07-06,2006-07-09 "Limerick",2006-08-21,2006-08-27 -"Duesseldorf",2006-10-30,2006-11-05 +"D??sseldorf",2006-10-30,2006-11-05 "Leysin",2007-01-08,2007-01-14 "Hildesheim",2007-03-01,2007-03-05 +"Hildesheim",2007-03-18,2007-03-23 +"Bern",2007-10-22,2007-10-26 +"G??teborg",2007-11-19,2007-11-25 +"Leysin",2008-01-12,2008-01-19 +"Berlin",2008-05-17,2008-05-22 +"EuroPython/Vilnius",2008-07-10,2008-07-12 +"D??sseldorf",2008-08-05,2008-08-13 +"Wroclaw",2009-02-07,2009-02-14 +"Leysin",2009-04-14,2009-04-21 +"G??teborg",2009-08-18,2009-08-25 +"D??sseldorf",2009-11-06,2009-11-13 +"CERN",2010-07-05,2010-07-09 +"D??sseldorf",2010-10-25,2010-10-31 Modified: pypy/branch/out-of-line-guards/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/argument.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/argument.py Thu Dec 9 16:33:59 2010 @@ -402,9 +402,10 @@ space = self.space w_args = space.newtuple(self.arguments_w) w_kwds = space.newdict() - for i in range(len(self.keywords)): - space.setitem(w_kwds, space.wrap(self.keywords[i]), - self.keywords_w[i]) + if self.keywords is not None: + for i in range(len(self.keywords)): + space.setitem(w_kwds, space.wrap(self.keywords[i]), + self.keywords_w[i]) return w_args, w_kwds class ArgumentsForTranslation(Arguments): Modified: pypy/branch/out-of-line-guards/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/baseobjspace.py Thu Dec 9 16:33:59 2010 @@ -147,7 +147,7 @@ __already_enqueued_for_destruction = False - def _enqueue_for_destruction(self, space): + def _enqueue_for_destruction(self, space, call_user_del=True): """Put the object in the destructor queue of the space. At a later, safe point in time, UserDelAction will use space.userdel() to call the object's app-level __del__ method. @@ -160,7 +160,8 @@ return self.__already_enqueued_for_destruction = True self.clear_all_weakrefs() - space.user_del_action.register_dying_object(self) + if call_user_del: + space.user_del_action.register_dying_object(self) def _call_builtin_destructor(self): pass # method overridden in typedef.py @@ -861,14 +862,14 @@ def call_args_and_c_profile(self, frame, w_func, args): ec = self.getexecutioncontext() - ec.c_call_trace(frame, w_func) + ec.c_call_trace(frame, w_func, args) try: w_res = self.call_args(w_func, args) except OperationError, e: w_value = e.get_w_value(self) ec.c_exception_trace(frame, w_value) raise - ec.c_return_trace(frame, w_func) + ec.c_return_trace(frame, w_func, args) return w_res def call_method(self, w_obj, methname, *arg_w): Modified: pypy/branch/out-of-line-guards/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/executioncontext.py Thu Dec 9 16:33:59 2010 @@ -27,7 +27,6 @@ def __init__(self, space): self.space = space self.topframeref = jit.vref_None - self.framestackdepth = 0 # tracing: space.frame_trace_action.fire() must be called to ensure # that tracing occurs whenever self.w_tracefunc or self.is_tracing # is modified. @@ -54,9 +53,6 @@ return frame def enter(self, frame): - if self.framestackdepth > self.space.sys.recursionlimit: - raise self.space.prebuilt_recursion_error - self.framestackdepth += 1 frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) @@ -66,7 +62,6 @@ self._trace(frame, 'leaveframe', self.space.w_None) finally: self.topframeref = frame.f_backref - self.framestackdepth -= 1 jit.virtual_ref_finish(frame) if self.w_tracefunc is not None and not frame.hide(): @@ -80,7 +75,6 @@ def __init__(self): self.topframe = None - self.framestackdepth = 0 self.w_tracefunc = None self.profilefunc = None self.w_profilefuncarg = None @@ -88,7 +82,6 @@ def enter(self, ec): ec.topframeref = jit.non_virtual_ref(self.topframe) - ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc ec.w_profilefuncarg = self.w_profilefuncarg @@ -97,7 +90,6 @@ def leave(self, ec): self.topframe = ec.gettopframe() - self.framestackdepth = ec.framestackdepth self.w_tracefunc = ec.w_tracefunc self.profilefunc = ec.profilefunc self.w_profilefuncarg = ec.w_profilefuncarg @@ -105,7 +97,6 @@ def clear_framestack(self): self.topframe = None - self.framestackdepth = 0 # the following interface is for pickling and unpickling def getstate(self, space): @@ -121,33 +112,41 @@ self.topframe = space.interp_w(PyFrame, frames_w[-1]) else: self.topframe = None - self.framestackdepth = len(frames_w) def getframestack(self): - index = self.framestackdepth - lst = [None] * index + lst = [] f = self.topframe - while index > 0: - index -= 1 - lst[index] = f + while f is not None: + lst.append(f) f = f.f_backref() - assert f is None + lst.reverse() return lst # coroutine: I think this is all, folks! - def c_call_trace(self, frame, w_func): + def c_call_trace(self, frame, w_func, args=None): "Profile the call of a builtin function" - if self.profilefunc is None: - frame.is_being_profiled = False - else: - self._trace(frame, 'c_call', w_func) + self._c_call_return_trace(frame, w_func, args, 'c_call') - def c_return_trace(self, frame, w_retval): + def c_return_trace(self, frame, w_func, args=None): "Profile the return from a builtin function" + self._c_call_return_trace(frame, w_func, args, 'c_return') + + def _c_call_return_trace(self, frame, w_func, args, event): if self.profilefunc is None: frame.is_being_profiled = False else: - self._trace(frame, 'c_return', w_retval) + # undo the effect of the CALL_METHOD bytecode, which would be + # that even on a built-in method call like '[].append()', + # w_func is actually the unbound function 'append'. + from pypy.interpreter.function import FunctionWithFixedCode + if isinstance(w_func, FunctionWithFixedCode) and args is not None: + w_firstarg = args.firstarg() + if w_firstarg is not None: + from pypy.interpreter.function import descr_function_get + w_func = descr_function_get(self.space, w_func, w_firstarg, + self.space.type(w_firstarg)) + # + self._trace(frame, event, w_func) def c_exception_trace(self, frame, w_exc): "Profile function called upon OperationError." Modified: pypy/branch/out-of-line-guards/pypy/interpreter/function.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/function.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/function.py Thu Dec 9 16:33:59 2010 @@ -619,11 +619,9 @@ self.w_func_dict = func.w_func_dict self.w_module = func.w_module - def descr_builtinfunction__new__(space, w_subtype, w_func): - func = space.interp_w(Function, w_func) - bltin = space.allocate_instance(BuiltinFunction, w_subtype) - BuiltinFunction.__init__(bltin, func) - return space.wrap(bltin) + def descr_builtinfunction__new__(space, w_subtype): + raise OperationError(space.w_TypeError, + space.wrap("cannot create 'builtin_function' instances")) def descr_function_repr(self): return self.space.wrap('' % (self.name,)) Modified: pypy/branch/out-of-line-guards/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/gateway.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/gateway.py Thu Dec 9 16:33:59 2010 @@ -1083,7 +1083,7 @@ # these decorators are known to return the same function # object, we may ignore them assert '\n' in source - source = source[source.find('\n') + 1:] + source = source[source.find('\n') + 1:].lstrip() assert source.startswith("def "), "can only transform functions" source = source[4:] p = source.find('(') Modified: pypy/branch/out-of-line-guards/pypy/interpreter/generator.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/generator.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/generator.py Thu Dec 9 16:33:59 2010 @@ -10,7 +10,7 @@ def __init__(self, frame): self.space = frame.space - self.frame = frame + self.frame = frame # turned into None when frame_finished_execution self.running = False def descr__reduce__(self, space): @@ -19,9 +19,13 @@ mod = space.interp_w(MixedModule, w_mod) new_inst = mod.get('generator_new') w = space.wrap + if self.frame: + w_frame = w(self.frame) + else: + w_frame = space.w_None tup = [ - w(self.frame), + w_frame, w(self.running), ] @@ -41,7 +45,8 @@ if self.running: raise OperationError(space.w_ValueError, space.wrap('generator already executing')) - if self.frame.frame_finished_execution: + frame = self.frame + if frame is None: # xxx a bit ad-hoc, but we don't want to go inside # execute_generator_frame() if the frame is actually finished if operr is None: @@ -49,7 +54,7 @@ raise operr # XXX it's not clear that last_instr should be promoted at all # but as long as it is necessary for call_assembler, let's do it early - last_instr = jit.hint(self.frame.last_instr, promote=True) + last_instr = jit.hint(frame.last_instr, promote=True) if last_instr == -1: if w_arg and not space.is_w(w_arg, space.w_None): msg = "can't send non-None value to a just-started generator" @@ -60,18 +65,19 @@ self.running = True try: try: - w_result = self.frame.execute_generator_frame(w_arg, operr) + w_result = frame.execute_generator_frame(w_arg, operr) except OperationError: # errors finish a frame - self.frame.frame_finished_execution = True + self.frame = None raise # if the frame is now marked as finished, it was RETURNed from - if self.frame.frame_finished_execution: + if frame.frame_finished_execution: + self.frame = None raise OperationError(space.w_StopIteration, space.w_None) else: return w_result # YIELDed finally: - self.frame.f_backref = jit.vref_None + frame.f_backref = jit.vref_None self.running = False def descr_throw(self, w_type, w_val=None, w_tb=None): @@ -115,7 +121,7 @@ raise OperationError(space.w_RuntimeError, space.wrap(msg)) def descr_gi_frame(space, self): - if not self.frame.frame_finished_execution: + if self.frame is not None and not self.frame.frame_finished_execution: return self.frame else: return space.w_None @@ -125,15 +131,17 @@ applevel __del__, which is called at a safe point after the interp-level __del__ enqueued the object for destruction """ - # Only bother raising an exception if the frame is still not - # finished and finally or except blocks are present. - if not self.frame.frame_finished_execution: + self.descr_close() + + def __del__(self): + # Only bother enqueuing self to raise an exception if the frame is + # still not finished and finally or except blocks are present. + must_call_close = False + if self.frame is not None: block = self.frame.lastblock while block is not None: if not isinstance(block, LoopBlock): - self.descr_close() - return + must_call_close = True + break block = block.previous - - def __del__(self): - self._enqueue_for_destruction(self.space) + self._enqueue_for_destruction(self.space, must_call_close) Modified: pypy/branch/out-of-line-guards/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/mixedmodule.py Thu Dec 9 16:33:59 2010 @@ -13,6 +13,10 @@ applevel_name = None expose__file__attribute = True + + # The following attribute is None as long as the module has not been + # imported yet, and when it has been, it is mod.__dict__.items() just + # after startup(). w_initialdict = None def __init__(self, space, w_name): @@ -26,8 +30,14 @@ """This is called each time the module is imported or reloaded """ if self.w_initialdict is not None: + # the module was already imported. Refresh its content with + # the saved dict, as done with built-in and extension modules + # on CPython. space.call_method(self.w_dict, 'update', self.w_initialdict) - Module.init(self, space) + else: + Module.init(self, space) + if not self.lazy and self.w_initialdict is None: + self.w_initialdict = space.call_method(self.w_dict, 'items') def get_applevel_name(cls): """ NOT_RPYTHON """ @@ -96,6 +106,7 @@ def _freeze_(self): self.getdict() + self.w_initialdict = None self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant Modified: pypy/branch/out-of-line-guards/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/pycode.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/pycode.py Thu Dec 9 16:33:59 2010 @@ -15,7 +15,7 @@ CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask -from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified +from pypy.rlib.debug import make_sure_not_resized from pypy.rlib import jit from pypy.rlib.objectmodel import compute_hash from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT @@ -69,7 +69,7 @@ self.co_stacksize = stacksize self.co_flags = flags self.co_code = code - self.co_consts_w = make_sure_not_modified(consts) + self.co_consts_w = consts self.co_names_w = [space.new_interned_str(aname) for aname in names] self.co_varnames = varnames self.co_freevars = freevars @@ -269,7 +269,7 @@ dis.dis(co) def fget_co_consts(space, self): - return space.newtuple(self.co_consts_w[:]) + return space.newtuple(self.co_consts_w) def fget_co_names(space, self): return space.newtuple(self.co_names_w) @@ -383,7 +383,7 @@ w(self.co_stacksize), w(self.co_flags), w(self.co_code), - space.newtuple(self.co_consts_w[:]), + space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), Modified: pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/error.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/error.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/error.py Thu Dec 9 16:33:59 2010 @@ -2,19 +2,22 @@ class SyntaxError(Exception): """Base class for exceptions raised by the parser.""" - def __init__(self, msg, lineno=0, offset=0, text=None, filename=None): + def __init__(self, msg, lineno=0, offset=0, text=None, filename=None, + lastlineno=0): self.msg = msg self.lineno = lineno self.offset = offset self.text = text self.filename = filename + self.lastlineno = lastlineno def wrap_info(self, space): return space.newtuple([space.wrap(self.msg), space.newtuple([space.wrap(self.filename), space.wrap(self.lineno), space.wrap(self.offset), - space.wrap(self.text)])]) + space.wrap(self.text), + space.wrap(self.lastlineno)])]) def __str__(self): return "%s at pos (%d, %d) in %r" % (self.__class__.__name__, @@ -33,8 +36,9 @@ class TokenError(SyntaxError): - def __init__(self, msg, line, lineno, column, tokens): - SyntaxError.__init__(self, msg, lineno, column, line) + def __init__(self, msg, line, lineno, column, tokens, lastlineno=0): + SyntaxError.__init__(self, msg, lineno, column, line, + lastlineno=lastlineno) self.tokens = tokens class TokenIndentationError(IndentationError): Modified: pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/pyparser/pytokenizer.py Thu Dec 9 16:33:59 2010 @@ -78,6 +78,7 @@ contline = None indents = [0] last_comment = '' + parenlevstart = (0, 0, "") # make the annotator happy endDFA = automata.DFA([], []) @@ -85,7 +86,7 @@ line = '' pos = 0 lines.append("") - strstart = (0, 0) + strstart = (0, 0, "") for line in lines: lnum = lnum + 1 pos, max = 0, len(line) @@ -93,7 +94,8 @@ if contstr: if not line: raise TokenError("EOF while scanning triple-quoted string", - line, lnum-1, 0, token_list) + strstart[2], strstart[0], strstart[1]+1, + token_list, lnum) endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch @@ -146,6 +148,10 @@ else: # continued statement if not line: + if parenlev > 0: + lnum1, start1, line1 = parenlevstart + raise TokenError("parenthesis is never closed", line1, + lnum1, start1 + 1, token_list, lnum) raise TokenError("EOF in multi-line statement", line, lnum, 0, token_list) continued = 0 @@ -187,7 +193,7 @@ token_list.append(tok) last_comment = '' else: - strstart = (lnum, start) + strstart = (lnum, start, line) contstr = line[start:] contline = line break @@ -195,7 +201,7 @@ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string - strstart = (lnum, start) + strstart = (lnum, start, line) endDFA = (endDFAs[initial] or endDFAs[token[1]] or endDFAs[token[2]]) contstr, needcont = line[start:], 1 @@ -212,6 +218,8 @@ continued = 1 else: if initial in '([{': + if parenlev == 0: + parenlevstart = (lnum, start, line) parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 @@ -230,7 +238,7 @@ start = pos if start', 'exec', 0) Modified: pypy/branch/out-of-line-guards/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/test/test_executioncontext.py Thu Dec 9 16:33:59 2010 @@ -7,6 +7,10 @@ class TestExecutionContext: + keywords = {} + + def setup_class(cls): + cls.space = gettestobjspace(**cls.keywords) def test_action(self): @@ -77,29 +81,43 @@ assert l == ['call', 'return', 'call', 'return'] def test_llprofile_c_call(self): + from pypy.interpreter.function import Function, Method l = [] + seen = [] + space = self.space - def profile_func(space, w_arg, frame, event, w_aarg): + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) + if event == 'c_call': + seen.append(w_func) - space = self.space - space.getexecutioncontext().setllprofile(profile_func, space.w_None) - - def check_snippet(snippet): + def check_snippet(snippet, expected_c_call): + del l[:] + del seen[:] + space.getexecutioncontext().setllprofile(profile_func, + space.w_None) space.appexec([], """(): %s return """ % snippet) space.getexecutioncontext().setllprofile(None, None) assert l == ['call', 'return', 'call', 'c_call', 'c_return', 'return'] - - check_snippet('l = []; l.append(42)') - check_snippet('max(1, 2)') - check_snippet('args = (1, 2); max(*args)') - check_snippet('max(1, 2, **{})') - check_snippet('args = (1, 2); max(*args, **{})') - check_snippet('abs(val=0)') + if isinstance(seen[0], Method): + found = 'method %s of %s' % ( + seen[0].w_function.name, + seen[0].w_class.getname(space, '?')) + else: + assert isinstance(seen[0], Function) + found = 'builtin %s' % seen[0].name + assert found == expected_c_call + + check_snippet('l = []; l.append(42)', 'method append of list') + check_snippet('max(1, 2)', 'builtin max') + check_snippet('args = (1, 2); max(*args)', 'builtin max') + check_snippet('max(1, 2, **{})', 'builtin max') + check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') + check_snippet('abs(val=0)', 'builtin abs') def test_llprofile_c_exception(self): l = [] @@ -243,6 +261,13 @@ """) +class TestExecutionContextWithCallLikelyBuiltin(TestExecutionContext): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True} + +class TestExecutionContextWithCallMethod(TestExecutionContext): + keywords = {'objspace.opcodes.CALL_METHOD': True} + + class AppTestDelNotBlocked: def setup_method(self, meth): Modified: pypy/branch/out-of-line-guards/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/test/test_function.py Thu Dec 9 16:33:59 2010 @@ -6,7 +6,7 @@ from pypy.interpreter.argument import Arguments -class AppTestFunctionIntrospection: +class AppTestFunctionIntrospection: def test_attributes(self): globals()['__name__'] = 'mymodulename' def f(): pass @@ -88,10 +88,10 @@ def f(*args): return 42 raises(TypeError, "dir.func_code = f.func_code") - raises(TypeError, "list.append.im_func.func_code = f.func_code") + raises(TypeError, "list.append.im_func.func_code = f.func_code") -class AppTestFunction: +class AppTestFunction: def test_simple_call(self): def func(arg1, arg2): return arg1, arg2 @@ -116,7 +116,7 @@ assert res[2] == 333 raises(TypeError, func) - raises(TypeError, func, 1, 2, 3, 4) + raises(TypeError, func, 1, 2, 3, 4) def test_simple_varargs(self): def func(arg1, *args): @@ -127,7 +127,7 @@ res = func(23, *(42,)) assert res[0] == 23 - assert res[1] == (42,) + assert res[1] == (42,) def test_simple_kwargs(self): def func(arg1, **kwargs): @@ -205,7 +205,7 @@ func(**{'self': 23}) assert False except TypeError: - pass + pass def test_kwargs_confusing_name(self): def func(self): # 'self' conflicts with the interp-level @@ -287,7 +287,7 @@ # on function types raises(ValueError, type(f).__setstate__, f, (1, 2, 3)) -class AppTestMethod: +class AppTestMethod: def test_simple_call(self): class A(object): def func(self, arg2): @@ -308,7 +308,7 @@ res = a.func(*(42,)) assert res[0] is a - assert res[1] == (42,) + assert res[1] == (42,) def test_obscure_varargs(self): class A(object): @@ -321,14 +321,14 @@ res = a.func(*(42,)) assert res[0] is a - assert res[1] == 42 + assert res[1] == 42 def test_simple_kwargs(self): class A(object): def func(self, **kwargs): return self, kwargs a = A() - + res = a.func(value=42) assert res[0] is a assert res[1] == {'value': 42} @@ -382,19 +382,19 @@ assert hash(C.m) == hash(D.m) assert hash(c.m) == hash(c.m) - def test_method_repr(self): - class A(object): - def f(self): + def test_method_repr(self): + class A(object): + def f(self): pass assert repr(A.f) == "" - assert repr(A().f).startswith(">") + assert repr(A().f).startswith(">") class B: def f(self): pass assert repr(B.f) == "" assert repr(B().f).startswith(">") + assert repr(A().f).endswith(">>") def test_method_call(self): @@ -487,14 +487,21 @@ def f(): pass raises(TypeError, new.instancemethod, f, None) + def test_empty_arg_kwarg_call(self): + def f(): + pass -class TestMethod: + raises(TypeError, lambda: f(*0)) + raises(TypeError, lambda: f(**0)) + + +class TestMethod: def setup_method(self, method): def c(self, bar): return bar code = PyCode._from_code(self.space, c.func_code) self.fn = Function(self.space, code, self.space.newdict()) - + def test_get(self): space = self.space w_meth = descr_function_get(space, self.fn, space.wrap(5), space.type(space.wrap(5))) @@ -552,7 +559,7 @@ def test_call_function(self): space = self.space - + d = {} for i in range(10): args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")" @@ -574,14 +581,14 @@ code.funcrun = bomb code.funcrun_obj = bomb - args_w = map(space.wrap, range(i)) + args_w = map(space.wrap, range(i)) w_res = space.call_function(fn, *args_w) check = space.is_true(space.eq(w_res, space.wrap(res))) assert check def test_flatcall(self): space = self.space - + def f(a): return a code = PyCode._from_code(self.space, f.func_code) @@ -608,7 +615,7 @@ def test_flatcall_method(self): space = self.space - + def f(self, a): return a code = PyCode._from_code(self.space, f.func_code) @@ -636,7 +643,7 @@ def test_flatcall_default_arg(self): space = self.space - + def f(a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -665,7 +672,7 @@ def test_flatcall_default_arg_method(self): space = self.space - + def f(self, a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -688,7 +695,7 @@ y = A().m(x) b = A().m z = b(x) - return y+10*z + return y+10*z """) assert space.eq_w(w_res, space.wrap(44)) Modified: pypy/branch/out-of-line-guards/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/out-of-line-guards/pypy/interpreter/test/test_gateway.py Thu Dec 9 16:33:59 2010 @@ -578,6 +578,11 @@ w_res = space.call_args(w_g, args) assert space.eq_w(w_res, space.wrap((-1, 0))) +class AppTestPyTestMark: + @py.test.mark.unlikely_to_exist + def test_anything(self): + pass + class TestPassThroughArguments: Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/detect_cpu.py Thu Dec 9 16:33:59 2010 @@ -31,7 +31,8 @@ 'i86pc': 'x86', # Solaris/Intel 'x86': 'x86', # Apple 'Power Macintosh': 'ppc', - 'x86_64': 'x86', + 'x86_64': 'x86', + 'amd64': 'x86' # freebsd }[mach] except KeyError: return mach Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py Thu Dec 9 16:33:59 2010 @@ -4,6 +4,7 @@ when executing on top of the llinterpreter. """ +import weakref from pypy.objspace.flow.model import Variable, Constant from pypy.annotation import model as annmodel from pypy.jit.metainterp.history import (ConstInt, ConstPtr, @@ -161,6 +162,8 @@ # ____________________________________________________________ class CompiledLoop(object): + has_been_freed = False + def __init__(self): self.inputargs = [] self.operations = [] @@ -285,6 +288,11 @@ del _variables[:] return _to_opaque(CompiledLoop()) +def mark_as_free(loop): + loop = _from_opaque(loop) + assert not loop.has_been_freed + loop.has_been_freed = True + def compile_start_int_var(loop): return compile_start_ref_var(loop, lltype.Signed) @@ -317,7 +325,7 @@ raise ValueError("CALL_ASSEMBLER not supported") loop = _from_opaque(loop) op = loop.operations[-1] - op.descr = descr + op.descr = weakref.ref(descr) def compile_add_var(loop, intvar): loop = _from_opaque(loop) @@ -429,6 +437,7 @@ verbose = True self.opindex = 0 while True: + assert not self.loop.has_been_freed op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): @@ -440,7 +449,10 @@ _stats.exec_conditional_jumps += 1 if op.jump_target is not None: # a patched guard, pointing to further code - args = [self.getenv(v) for v in op.fail_args if v] + if op.fail_args: + args = [self.getenv(v) for v in op.fail_args if v] + else: + args = [] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target @@ -571,7 +583,12 @@ def op_debug_merge_point(self, _, value, recdepth): from pypy.jit.metainterp.warmspot import get_stats loc = ConstPtr(value)._get_str() - get_stats().add_merge_point_location(loc) + try: + stats = get_stats() + except AttributeError: + pass + else: + stats.add_merge_point_location(loc) def op_guard_true(self, _, value): if not value: @@ -839,14 +856,22 @@ finally: self._may_force = -1 - def op_call_assembler(self, loop_token, *args): + def op_call_assembler(self, wref_loop_token, *args): + if we_are_translated(): + raise ValueError("CALL_ASSEMBLER not supported") + return self._do_call_assembler(wref_loop_token, *args) + + def _do_call_assembler(self, wref_loop_token, *args): global _last_exception + loop_token = wref_loop_token() + assert loop_token, "CALL_ASSEMBLER to a target that already died" + ctl = loop_token.compiled_loop_token + if hasattr(ctl, 'redirected'): + return self._do_call_assembler(ctl.redirected, *args) assert not self._forced - loop_token = self.cpu._redirected_call_assembler.get(loop_token, - loop_token) self._may_force = self.opindex try: - inpargs = _from_opaque(loop_token._llgraph_compiled_version).inputargs + inpargs = _from_opaque(ctl.compiled_version).inputargs for i, inparg in enumerate(inpargs): TYPE = inparg.concretetype if TYPE is lltype.Signed: @@ -1539,10 +1564,13 @@ do_setfield_gc_int(vable, fielddescr.ofs, 0) def redirect_call_assembler(cpu, oldlooptoken, newlooptoken): - OLD = _from_opaque(oldlooptoken._llgraph_compiled_version).getargtypes() - NEW = _from_opaque(newlooptoken._llgraph_compiled_version).getargtypes() + oldclt = oldlooptoken.compiled_loop_token + newclt = newlooptoken.compiled_loop_token + OLD = _from_opaque(oldclt.compiled_version).getargtypes() + NEW = _from_opaque(newclt.compiled_version).getargtypes() assert OLD == NEW - cpu._redirected_call_assembler[oldlooptoken] = newlooptoken + assert not hasattr(oldclt, 'redirected') + oldclt.redirected = weakref.ref(newlooptoken) # ____________________________________________________________ @@ -1609,6 +1637,7 @@ setannotation(compile_add_fail, annmodel.SomeInteger()) setannotation(compile_add_fail_arg, annmodel.s_None) setannotation(compile_redirect_fail, annmodel.s_None) +setannotation(mark_as_free, annmodel.s_None) setannotation(new_frame, s_Frame) setannotation(frame_clear, annmodel.s_None) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py Thu Dec 9 16:33:59 2010 @@ -102,7 +102,6 @@ llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] self._descrs = {} - self._redirected_call_assembler = {} def _freeze_(self): assert self.translate_support_code @@ -118,22 +117,34 @@ self._descrs[key] = descr return descr - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): c = llimpl.compile_start() + clt = original_loop_token.compiled_loop_token + clt.loop_and_bridges.append(c) + clt.compiling_a_bridge() self._compile_loop_or_bridge(c, inputargs, operations) old, oldindex = faildescr._compiled_fail llimpl.compile_redirect_fail(old, oldindex, c) - def compile_loop(self, inputargs, operations, loopdescr, log=True): + def compile_loop(self, inputargs, operations, looptoken, log=True): """In a real assembler backend, this should assemble the given list of operations. Here we just generate a similar CompiledLoop instance. The code here is RPython, whereas the code in llimpl is not. """ c = llimpl.compile_start() - loopdescr._llgraph_compiled_version = c + clt = model.CompiledLoopToken(self, looptoken.number) + clt.loop_and_bridges = [c] + clt.compiled_version = c + looptoken.compiled_loop_token = clt self._compile_loop_or_bridge(c, inputargs, operations) + def free_loop_and_bridges(self, compiled_loop_token): + for c in compiled_loop_token.loop_and_bridges: + llimpl.mark_as_free(c) + model.AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) + def _compile_loop_or_bridge(self, c, inputargs, operations): var2index = {} for box in inputargs: @@ -207,7 +218,7 @@ if op.getopnum() == rop.JUMP: targettoken = op.getdescr() assert isinstance(targettoken, history.LoopToken) - compiled_version = targettoken._llgraph_compiled_version + compiled_version = targettoken.compiled_loop_token.compiled_version llimpl.compile_add_jump_target(c, compiled_version) elif op.getopnum() == rop.FINISH: faildescr = op.getdescr() @@ -217,7 +228,7 @@ assert False, "unknown operation" def _execute_token(self, loop_token): - compiled_version = loop_token._llgraph_compiled_version + compiled_version = loop_token.compiled_loop_token.compiled_version frame = llimpl.new_frame(self.is_oo, self) # setup the frame llimpl.frame_clear(frame, compiled_version) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/gc.py Thu Dec 9 16:33:59 2010 @@ -15,6 +15,7 @@ from pypy.jit.backend.llsupport.descr import GcCache, get_field_descr from pypy.jit.backend.llsupport.descr import GcPtrFieldDescr from pypy.jit.backend.llsupport.descr import get_call_descr +from pypy.rpython.memory.gctransform import asmgcroot # ____________________________________________________________ @@ -35,6 +36,8 @@ return False def has_write_barrier_class(self): return None + def freeing_block(self, start, stop): + pass # ____________________________________________________________ @@ -218,50 +221,120 @@ LOC_EBP_PLUS = 2 LOC_EBP_MINUS = 3 - GCMAP_ARRAY = rffi.CArray(llmemory.Address) - CALLSHAPE_ARRAY = rffi.CArray(rffi.UCHAR) + GCMAP_ARRAY = rffi.CArray(lltype.Signed) + CALLSHAPE_ARRAY_PTR = rffi.CArrayPtr(rffi.UCHAR) def __init__(self): + # '_gcmap' is an array of length '_gcmap_maxlength' of addresses. + # '_gcmap_curlength' tells how full the array really is. + # The addresses are actually grouped in pairs: + # (addr-after-the-CALL-in-assembler, addr-of-the-call-shape). + # '_gcmap_deadentries' counts pairs marked dead (2nd item is NULL). + # '_gcmap_sorted' is True only if we know the array is sorted. self._gcmap = lltype.nullptr(self.GCMAP_ARRAY) self._gcmap_curlength = 0 self._gcmap_maxlength = 0 + self._gcmap_deadentries = 0 + self._gcmap_sorted = True def initialize(self): # hack hack hack. Remove these lines and see MissingRTypeAttribute # when the rtyper tries to annotate these methods only when GC-ing... self.gcmapstart() self.gcmapend() + self.gcmarksorted() def gcmapstart(self): - return llmemory.cast_ptr_to_adr(self._gcmap) + return rffi.cast(llmemory.Address, self._gcmap) def gcmapend(self): addr = self.gcmapstart() if self._gcmap_curlength: - addr += llmemory.sizeof(llmemory.Address)*self._gcmap_curlength + addr += rffi.sizeof(lltype.Signed) * self._gcmap_curlength + if not we_are_translated() and type(addr) is long: + from pypy.rpython.lltypesystem import ll2ctypes + addr = ll2ctypes._lladdress(addr) # XXX workaround return addr + def gcmarksorted(self): + # Called by the GC when it is about to sort [gcmapstart():gcmapend()]. + # Returns the previous sortedness flag -- i.e. returns True if it + # is already sorted, False if sorting is needed. + sorted = self._gcmap_sorted + self._gcmap_sorted = True + return sorted + def put(self, retaddr, callshapeaddr): """'retaddr' is the address just after the CALL. - 'callshapeaddr' is the address returned by encode_callshape().""" + 'callshapeaddr' is the address of the raw 'shape' marker. + Both addresses are actually integers here.""" index = self._gcmap_curlength if index + 2 > self._gcmap_maxlength: - self._enlarge_gcmap() + index = self._enlarge_gcmap() self._gcmap[index] = retaddr self._gcmap[index+1] = callshapeaddr self._gcmap_curlength = index + 2 + self._gcmap_sorted = False + @rgc.no_collect def _enlarge_gcmap(self): - newlength = 250 + self._gcmap_maxlength * 2 - newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', - track_allocation=False) oldgcmap = self._gcmap - for i in range(self._gcmap_curlength): - newgcmap[i] = oldgcmap[i] - self._gcmap = newgcmap - self._gcmap_maxlength = newlength - if oldgcmap: - lltype.free(oldgcmap, flavor='raw', track_allocation=False) + if self._gcmap_deadentries * 3 * 2 > self._gcmap_maxlength: + # More than 1/3rd of the entries are dead. Don't actually + # enlarge the gcmap table, but just clean up the dead entries. + newgcmap = oldgcmap + else: + # Normal path: enlarge the array. + newlength = 250 + (self._gcmap_maxlength // 3) * 4 + newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', + track_allocation=False) + self._gcmap_maxlength = newlength + # + j = 0 + i = 0 + end = self._gcmap_curlength + while i < end: + if oldgcmap[i + 1]: + newgcmap[j] = oldgcmap[i] + newgcmap[j + 1] = oldgcmap[i + 1] + j += 2 + i += 2 + self._gcmap_curlength = j + self._gcmap_deadentries = 0 + if oldgcmap != newgcmap: + self._gcmap = newgcmap + if oldgcmap: + lltype.free(oldgcmap, flavor='raw', track_allocation=False) + return j + + @rgc.no_collect + def freeing_block(self, start, stop): + # if [start:stop] is a raw block of assembler, then look up the + # corresponding gcroot markers, and mark them as freed now in + # self._gcmap by setting the 2nd address of every entry to NULL. + gcmapstart = self.gcmapstart() + gcmapend = self.gcmapend() + if gcmapstart == gcmapend: + return + if not self.gcmarksorted(): + asmgcroot.sort_gcmap(gcmapstart, gcmapend) + # A note about gcmarksorted(): the deletion we do here keeps the + # array sorted. This avoids needing too many sort_gcmap()s. + # Indeed, freeing_block() is typically called many times in a row, + # so it will call sort_gcmap() at most the first time. + startaddr = rffi.cast(llmemory.Address, start) + stopaddr = rffi.cast(llmemory.Address, stop) + item = asmgcroot.binary_search(gcmapstart, gcmapend, startaddr) + # 'item' points to one of the entries. Because the whole array + # is sorted, we know that it points either to the first entry we + # want to kill, or to the previous entry. + if item.address[0] < startaddr: + item += asmgcroot.arrayitemsize # go forward one entry + assert item == gcmapend or item.address[0] >= startaddr + while item != gcmapend and item.address[0] < stopaddr: + item.address[1] = llmemory.NULL + self._gcmap_deadentries += 1 + item += asmgcroot.arrayitemsize def get_basic_shape(self, is_64_bit=False): # XXX: Should this code even really know about stack frame layout of @@ -304,17 +377,15 @@ assert reg_index > 0 shape.append(chr(self.LOC_REG | (reg_index << 2))) - def compress_callshape(self, shape): + def compress_callshape(self, shape, datablockwrapper): # Similar to compress_callshape() in trackgcroot.py. - # XXX so far, we always allocate a new small array (we could regroup - # them inside bigger arrays) and we never try to share them. + # Returns an address to raw memory (as an integer). length = len(shape) - compressed = lltype.malloc(self.CALLSHAPE_ARRAY, length, - flavor='raw', - track_allocation=False) # memory leak + rawaddr = datablockwrapper.malloc_aligned(length, 1) + p = rffi.cast(self.CALLSHAPE_ARRAY_PTR, rawaddr) for i in range(length): - compressed[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) - return llmemory.cast_ptr_to_adr(compressed) + p[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) + return rawaddr class WriteBarrierDescr(AbstractDescr): @@ -379,6 +450,7 @@ 'layoutbuilder': self.layoutbuilder, 'gcmapstart': lambda: gcrootmap.gcmapstart(), 'gcmapend': lambda: gcrootmap.gcmapend(), + 'gcmarksorted': lambda: gcrootmap.gcmarksorted(), } self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc @@ -641,6 +713,9 @@ def has_write_barrier_class(self): return WriteBarrierDescr + def freeing_block(self, start, stop): + self.gcrootmap.freeing_block(start, stop) + # ____________________________________________________________ def get_ll_description(gcdescr, translator=None, rtyper=None): Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/llmodel.py Thu Dec 9 16:33:59 2010 @@ -17,7 +17,7 @@ from pypy.jit.backend.llsupport.descr import get_call_descr from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr -from pypy.jit.backend.llsupport.ffisupport import get_call_descr_dynamic +from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -52,6 +52,7 @@ else: self._setup_exception_handling_untranslated() self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) + self.asmmemmgr = AsmMemoryManager() self.setup() if translate_support_code: self._setup_on_leave_jitted_translated() @@ -177,6 +178,15 @@ self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) return exc + def free_loop_and_bridges(self, compiled_loop_token): + AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) + blocks = compiled_loop_token.asmmemmgr_blocks + if blocks is not None: + compiled_loop_token.asmmemmgr_blocks = None + for rawstart, rawstop in blocks: + self.gc_ll_descr.freeing_block(rawstart, rawstop) + self.asmmemmgr.free(rawstart, rawstop) + # ------------------- helpers and descriptions -------------------- @staticmethod @@ -236,7 +246,9 @@ return get_call_descr(self.gc_ll_descr, ARGS, RESULT, extrainfo) def calldescrof_dynamic(self, ffi_args, ffi_result, extrainfo=None): - return get_call_descr_dynamic(ffi_args, ffi_result, extrainfo) + from pypy.jit.backend.llsupport import ffisupport + return ffisupport.get_call_descr_dynamic(ffi_args, ffi_result, + extrainfo) def get_overflow_error(self): ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llsupport/test/test_gc.py Thu Dec 9 16:33:59 2010 @@ -9,6 +9,7 @@ from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.jit.metainterp.test.test_optimizeopt import equaloplists +from pypy.rpython.memory.gctransform import asmgcroot def test_boehm(): gc_ll_descr = GcLLDescr_boehm(None, None, None) @@ -62,58 +63,169 @@ for i in range(len(allocs)): assert addrs[i].address[0] == llmemory.cast_ptr_to_adr(allocs[i]) -def test_GcRootMap_asmgcc(): - def frame_pos(n): - return -4*(4+n) - gcrootmap = GcRootMap_asmgcc() - num1 = frame_pos(-5) - num1a = num1|2 - num2 = frame_pos(55) - num2a = ((-num2|3) >> 7) | 128 - num2b = (-num2|3) & 127 - shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, num1) - gcrootmap.add_ebp_offset(shape, num2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) - gcrootmap.add_callee_save_reg(shape, 1) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4]) - gcrootmap.add_callee_save_reg(shape, 2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8]) - gcrootmap.add_callee_save_reg(shape, 3) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12]) - gcrootmap.add_callee_save_reg(shape, 4) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12, 16]) - # - shapeaddr = gcrootmap.compress_callshape(shape) - PCALLSHAPE = lltype.Ptr(GcRootMap_asmgcc.CALLSHAPE_ARRAY) - p = llmemory.cast_adr_to_ptr(shapeaddr, PCALLSHAPE) - for i, expected in enumerate([16, 12, 8, 4, - num2a, num2b, num1a, 0, 2, 15, 11, 7, 6]): - assert p[i] == expected - # - retaddr = rffi.cast(llmemory.Address, 1234567890) - gcrootmap.put(retaddr, shapeaddr) - assert gcrootmap._gcmap[0] == retaddr - assert gcrootmap._gcmap[1] == shapeaddr - assert gcrootmap.gcmapstart().address[0] == retaddr - # - # the same as before, but enough times to trigger a few resizes - expected_shapeaddr = {} - for i in range(1, 700): +class TestGcRootMapAsmGcc: + + def test_make_shapes(self): + def frame_pos(n): + return -4*(4+n) + gcrootmap = GcRootMap_asmgcc() + num1 = frame_pos(-5) + num1a = num1|2 + num2 = frame_pos(55) + num2a = ((-num2|3) >> 7) | 128 + num2b = (-num2|3) & 127 shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, frame_pos(i)) - shapeaddr = gcrootmap.compress_callshape(shape) - expected_shapeaddr[i] = shapeaddr - retaddr = rffi.cast(llmemory.Address, 123456789 + i) + gcrootmap.add_ebp_offset(shape, num1) + gcrootmap.add_ebp_offset(shape, num2) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) + gcrootmap.add_callee_save_reg(shape, 1) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4]) + gcrootmap.add_callee_save_reg(shape, 2) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8]) + gcrootmap.add_callee_save_reg(shape, 3) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8, 12]) + gcrootmap.add_callee_save_reg(shape, 4) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8, 12, 16]) + + def test_compress_callshape(self): + class FakeDataBlockWrapper: + def malloc_aligned(self, size, alignment): + assert alignment == 1 # here + assert size == 4 + return rffi.cast(lltype.Signed, p) + datablockwrapper = FakeDataBlockWrapper() + p = lltype.malloc(rffi.CArray(lltype.Char), 4, immortal=True) + gcrootmap = GcRootMap_asmgcc() + shape = ['a', 'b', 'c', 'd'] + gcrootmap.compress_callshape(shape, datablockwrapper) + assert p[0] == 'd' + assert p[1] == 'c' + assert p[2] == 'b' + assert p[3] == 'a' + + def test_put_basic(self): + gcrootmap = GcRootMap_asmgcc() + retaddr = 1234567890 + shapeaddr = 51627384 gcrootmap.put(retaddr, shapeaddr) - for i in range(1, 700): - expected_retaddr = rffi.cast(llmemory.Address, 123456789 + i) - assert gcrootmap._gcmap[i*2+0] == expected_retaddr - assert gcrootmap._gcmap[i*2+1] == expected_shapeaddr[i] + assert gcrootmap._gcmap[0] == retaddr + assert gcrootmap._gcmap[1] == shapeaddr + p = rffi.cast(rffi.LONGP, gcrootmap.gcmapstart()) + assert p[0] == retaddr + assert (gcrootmap.gcmapend() == + gcrootmap.gcmapstart() + rffi.sizeof(lltype.Signed) * 2) + + def test_put_resize(self): + # the same as before, but enough times to trigger a few resizes + gcrootmap = GcRootMap_asmgcc() + for i in range(700): + shapeaddr = i * 100 + 1 + retaddr = 123456789 + i + gcrootmap.put(retaddr, shapeaddr) + for i in range(700): + assert gcrootmap._gcmap[i*2+0] == 123456789 + i + assert gcrootmap._gcmap[i*2+1] == i * 100 + 1 + + def test_remove_nulls(self): + expected = [] + def check(): + assert gcrootmap._gcmap_curlength == len(expected) * 2 + for i, (a, b) in enumerate(expected): + assert gcrootmap._gcmap[i*2] == a + assert gcrootmap._gcmap[i*2+1] == b + # + gcrootmap = GcRootMap_asmgcc() + for i in range(700): + shapeaddr = i * 100 # 0 if i == 0 + retaddr = 123456789 + i + gcrootmap.put(retaddr, shapeaddr) + if shapeaddr != 0: + expected.append((retaddr, shapeaddr)) + # at the first resize, the 0 should be removed + check() + for repeat in range(10): + # now clear up half the entries + assert len(expected) == 699 + for i in range(0, len(expected), 2): + gcrootmap._gcmap[i*2+1] = 0 + gcrootmap._gcmap_deadentries += 1 + expected = expected[1::2] + assert gcrootmap._gcmap_deadentries*6 > gcrootmap._gcmap_maxlength + # check that we can again insert 350 entries without a resize + oldgcmap = gcrootmap._gcmap + for i in range(0, 699, 2): + gcrootmap.put(515151 + i + repeat, 626262 + i) + expected.append((515151 + i + repeat, 626262 + i)) + assert gcrootmap._gcmap == oldgcmap + check() + + def test_freeing_block(self): + from pypy.jit.backend.llsupport import gc + class Asmgcroot: + arrayitemsize = 2 * llmemory.sizeof(llmemory.Address) + sort_count = 0 + def sort_gcmap(self, gcmapstart, gcmapend): + self.sort_count += 1 + def binary_search(self, gcmapstart, gcmapend, startaddr): + i = 0 + while (i < gcrootmap._gcmap_curlength//2 and + gcrootmap._gcmap[i*2] < startaddr): + i += 1 + if i > 0: + i -= 1 + assert 0 <= i < gcrootmap._gcmap_curlength//2 + p = rffi.cast(rffi.CArrayPtr(llmemory.Address), gcmapstart) + p = rffi.ptradd(p, 2*i) + return llmemory.cast_ptr_to_adr(p) + saved = gc.asmgcroot + try: + gc.asmgcroot = Asmgcroot() + # + gcrootmap = GcRootMap_asmgcc() + gcrootmap._gcmap = lltype.malloc(gcrootmap.GCMAP_ARRAY, + 1400, flavor='raw', + immortal=True) + for i in range(700): + gcrootmap._gcmap[i*2] = 1200000 + i + gcrootmap._gcmap[i*2+1] = i * 100 + 1 + assert gcrootmap._gcmap_deadentries == 0 + assert gc.asmgcroot.sort_count == 0 + gcrootmap._gcmap_maxlength = 1400 + gcrootmap._gcmap_curlength = 1400 + gcrootmap._gcmap_sorted = False + # + gcrootmap.freeing_block(1200000 - 100, 1200000) + assert gcrootmap._gcmap_deadentries == 0 + assert gc.asmgcroot.sort_count == 1 + # + gcrootmap.freeing_block(1200000 + 100, 1200000 + 200) + assert gcrootmap._gcmap_deadentries == 100 + assert gc.asmgcroot.sort_count == 1 + for i in range(700): + if 100 <= i < 200: + expected = 0 + else: + expected = i * 100 + 1 + assert gcrootmap._gcmap[i*2] == 1200000 + i + assert gcrootmap._gcmap[i*2+1] == expected + # + gcrootmap.freeing_block(1200000 + 650, 1200000 + 750) + assert gcrootmap._gcmap_deadentries == 150 + assert gc.asmgcroot.sort_count == 1 + for i in range(700): + if 100 <= i < 200 or 650 <= i: + expected = 0 + else: + expected = i * 100 + 1 + assert gcrootmap._gcmap[i*2] == 1200000 + i + assert gcrootmap._gcmap[i*2+1] == expected + # + finally: + gc.asmgcroot = saved class FakeLLOp(object): Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/model.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/model.py Thu Dec 9 16:33:59 2010 @@ -1,3 +1,4 @@ +from pypy.rlib.debug import debug_start, debug_print, debug_stop from pypy.jit.metainterp import history, compile @@ -7,17 +8,27 @@ done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + total_compiled_loops = 0 + total_compiled_bridges = 0 + total_freed_loops = 0 + total_freed_bridges = 0 def __init__(self): self.fail_descr_list = [] + self.fail_descr_free_list = [] def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) n = descr.index if n < 0: lst = self.fail_descr_list - n = len(lst) - lst.append(descr) + if len(self.fail_descr_free_list) > 0: + n = self.fail_descr_free_list.pop() + assert lst[n] is None + lst[n] = descr + else: + n = len(lst) + lst.append(descr) descr.index = n return n @@ -35,12 +46,14 @@ def compile_loop(self, inputargs, operations, looptoken, log=True): """Assemble the given loop. - Extra attributes should be put in the LoopToken to - point to the compiled loop in assembler. + Should create and attach a fresh CompiledLoopToken to + looptoken.compiled_loop_token and stick extra attributes + on it to point to the compiled loop in assembler. """ raise NotImplementedError - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): """Assemble the bridge. The FailDescr is the descr of the original guard that failed. """ @@ -113,6 +126,28 @@ oldlooptoken so that from now own they will call newlooptoken.""" raise NotImplementedError + def free_loop_and_bridges(self, compiled_loop_token): + """This method is called to free resources (machine code, + references to resume guards, etc.) allocated by the compilation + of a loop and all bridges attached to it. After this call, the + frontend cannot use this compiled loop any more; in fact, it + guarantees that at the point of the call to free_code_group(), + none of the corresponding assembler is currently running. + """ + # The base class provides a limited implementation: freeing the + # resume descrs. This is already quite helpful, because the + # resume descrs are the largest consumers of memory (about 3x + # more than the assembler, in the case of the x86 backend). + lst = self.fail_descr_list + # We expect 'compiled_loop_token' to be itself garbage-collected soon, + # but better safe than sorry: be ready to handle several calls to + # free_loop_and_bridges() for the same compiled_loop_token. + faildescr_indices = compiled_loop_token.faildescr_indices + compiled_loop_token.faildescr_indices = [] + for n in faildescr_indices: + lst[n] = None + self.fail_descr_free_list.extend(faildescr_indices) + @staticmethod def sizeof(S): raise NotImplementedError @@ -237,3 +272,40 @@ def force(self, force_token): raise NotImplementedError + + +class CompiledLoopToken(object): + asmmemmgr_blocks = None + asmmemmgr_gcroots = 0 + + def __init__(self, cpu, number): + cpu.total_compiled_loops += 1 + self.cpu = cpu + self.number = number + self.bridges_count = 0 + # This growing list gives the 'descr_number' of all fail descrs + # that belong to this loop or to a bridge attached to it. + # Filled by the frontend calling record_faildescr_index(). + self.faildescr_indices = [] + debug_start("jit-mem-looptoken-alloc") + debug_print("allocating Loop #", self.number) + debug_stop("jit-mem-looptoken-alloc") + + def record_faildescr_index(self, n): + self.faildescr_indices.append(n) + + def compiling_a_bridge(self): + self.cpu.total_compiled_bridges += 1 + self.bridges_count += 1 + debug_start("jit-mem-looptoken-alloc") + debug_print("allocating Bridge #", self.bridges_count, "of Loop #", self.number) + debug_stop("jit-mem-looptoken-alloc") + + def __del__(self): + debug_start("jit-mem-looptoken-free") + debug_print("freeing Loop #", self.number, 'with', + self.bridges_count, 'attached bridges') + self.cpu.free_loop_and_bridges(self) + self.cpu.total_freed_loops += 1 + self.cpu.total_freed_bridges += self.bridges_count + debug_stop("jit-mem-looptoken-free") Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/test/runner_test.py Thu Dec 9 16:33:59 2010 @@ -174,6 +174,8 @@ assert not wr_i1() and not wr_guard() def test_compile_bridge(self): + self.cpu.total_compiled_loops = 0 + self.cpu.total_compiled_bridges = 0 i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() @@ -199,7 +201,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) @@ -207,6 +209,10 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + assert self.cpu.total_compiled_loops == 1 + assert self.cpu.total_compiled_bridges == 1 + return looptoken + def test_compile_bridge_with_holes(self): i0 = BoxInt() i1 = BoxInt() @@ -233,7 +239,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) @@ -1050,7 +1056,7 @@ ResOperation(rop.JUMP, [f3] + fboxes2[1:], None, descr=looptoken), ] - self.cpu.compile_bridge(faildescr1, fboxes2, bridge) + self.cpu.compile_bridge(faildescr1, fboxes2, bridge, looptoken) for i in range(len(fboxes)): self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) @@ -1197,6 +1203,13 @@ yield nan_and_infinity, rop.FLOAT_GT, operator.gt, all_cases_binary yield nan_and_infinity, rop.FLOAT_GE, operator.ge, all_cases_binary + def test_noops(self): + c_box = self.alloc_string("hi there").constbox() + c_nest = ConstInt(0) + self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest], 'void') + self.execute_operation(rop.JIT_DEBUG, [c_box, c_nest, c_nest, + c_nest, c_nest], 'void') + class LLtypeBackendTest(BaseBackendTest): @@ -2224,6 +2237,20 @@ assert res.value == expected, ( "%r: got %r, expected %r" % (RESTYPE, res.value, expected)) + def test_free_loop_and_bridges(self): + from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("not a subclass of llmodel.AbstractLLCPU") + if hasattr(self.cpu, 'setup_once'): + self.cpu.setup_once() + mem0 = self.cpu.asmmemmgr.total_mallocs + looptoken = self.test_compile_bridge() + mem1 = self.cpu.asmmemmgr.total_mallocs + self.cpu.free_loop_and_bridges(looptoken.compiled_loop_token) + mem2 = self.cpu.asmmemmgr.total_mallocs + assert mem2 < mem1 + assert mem2 == mem0 + class OOtypeBackendTest(BaseBackendTest): Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/test/test_random.py Thu Dec 9 16:33:59 2010 @@ -524,7 +524,7 @@ self.prebuilt_ptr_consts = [] self.r = r self.build_random_loop(cpu, builder_factory, r, startvars) - + def build_random_loop(self, cpu, builder_factory, r, startvars): loop = TreeLoop('test_random_function') @@ -685,11 +685,12 @@ subloop.operations[-1] = jump_op self.guard_op = rl.guard_op self.prebuilt_ptr_consts += rl.prebuilt_ptr_consts + self.loop.token.record_jump_to(rl.loop.token) self.dont_generate_more = True if r.random() < .05: return False self.builder.cpu.compile_bridge(fail_descr, fail_args, - subloop.operations) + subloop.operations, self.loop.token) return True def check_random_function(cpu, BuilderClass, r, num=None, max=None): Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/assembler.py Thu Dec 9 16:33:59 2010 @@ -1,12 +1,13 @@ import sys, os from pypy.jit.backend.llsupport import symbolic +from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT, LoopToken) from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper -from pypy.tool.uid import fixid +from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, _get_scale) @@ -30,10 +31,11 @@ from pypy.jit.backend.x86 import rx86, regloc, codebuf from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array -from pypy.rlib.debug import debug_print +from pypy.jit.backend.x86 import support +from pypy.rlib.debug import (debug_print, debug_start, debug_stop, + have_debug_prints) from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout -from pypy.rlib.streamio import open_file_as_stream from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, @@ -43,123 +45,17 @@ def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) -class MachineCodeBlockWrapper(object): - MC_DEFAULT_SIZE = 1024*1024 - - def __init__(self, assembler, bigsize, profile_agent=None): - self.assembler = assembler - self.old_mcs = [] # keepalive - self.bigsize = bigsize - self._mc = self._instantiate_mc() - self.function_name = None - self.profile_agent = profile_agent - self.reset_reserved_bytes() - - def _instantiate_mc(self): # hook for testing - return codebuf.MachineCodeBlock(self.bigsize) - - def ensure_bytes_available(self, num_bytes): - if self.bytes_free() <= (self._reserved_bytes + num_bytes): - self.make_new_mc() - - def reserve_bytes(self, num_bytes): - self.ensure_bytes_available(num_bytes) - self._reserved_bytes += num_bytes - - def reset_reserved_bytes(self): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - self._reserved_bytes = 64 - - def get_relative_pos(self): - return self._mc.get_relative_pos() - - def overwrite(self, pos, listofchars): - return self._mc.overwrite(pos, listofchars) - - def bytes_free(self): - return self._mc._size - self._mc.get_relative_pos() - - def start_function(self, name): - self.function_name = name - self.start_pos = self._mc.get_relative_pos() - - def end_function(self, done=True): - assert self.function_name is not None - size = self._mc.get_relative_pos() - self.start_pos - address = self.tell() - size - if self.profile_agent is not None: - self.profile_agent.native_code_written(self.function_name, - address, size) - if done: - self.function_name = None - - def make_new_mc(self): - new_mc = self._instantiate_mc() - debug_print('[new machine code block at', new_mc.tell(), ']') - - if IS_X86_64: - # The scratch register is sometimes used as a temporary - # register, but the JMP below might clobber it. Rather than risk - # subtle bugs, we preserve the scratch register across the jump. - self._mc.PUSH_r(X86_64_SCRATCH_REG.value) - - self._mc.JMP(imm(new_mc.tell())) - - if IS_X86_64: - # Restore scratch reg - new_mc.POP_r(X86_64_SCRATCH_REG.value) - - if self.function_name is not None: - self.end_function(done=False) - self.start_pos = new_mc.get_relative_pos() - - self.assembler.write_pending_failure_recoveries() - - self._mc.done() - self.old_mcs.append(self._mc) - self._mc = new_mc - make_new_mc._dont_inline_ = True - - def tell(self): - return self._mc.tell() - - def done(self): - self._mc.done() - -def _new_method(name): - def method(self, *args): - if self.bytes_free() < self._reserved_bytes: - self.make_new_mc() - getattr(self._mc, name)(*args) - method.func_name = name - return method - -for _name in rx86.all_instructions + regloc.all_extra_instructions: - setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) - -for name in dir(codebuf.MachineCodeBlock): - if name.upper() == name or name == "writechr": - setattr(MachineCodeBlockWrapper, name, _new_method(name)) class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + def __init__(self, faildescr, failargs, fail_locs, exc): self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs self.exc = exc - self.desc_bytes = desc_bytes - - def recovery_stub_size(self): - # XXX: 32 is pulled out of the air - return 32 + len(self.desc_bytes) DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) class Assembler386(object): - mc = None - mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE - _float_constants = None _regalloc = None _output_loop_log = None @@ -177,16 +73,18 @@ self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 self.loop_run_counters = [] - # if we have 10000 loops, we have some other problems I guess self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 - self.pending_guard_tokens = None self.memcpy_addr = 0 self.setup_failure_recovery() self._debug = False self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.fail_boxes_count = 0 + self._current_depths_cache = (0, 0) + self.datablockwrapper = None + self.teardown() def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -196,67 +94,65 @@ def set_debug(self, v): self._debug = v - def setup(self): - if self.mc is None: - # the address of the function called by 'new' - gc_ll_descr = self.cpu.gc_ll_descr - gc_ll_descr.initialize() - ll_new = gc_ll_descr.get_funcptr_for_new() - self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) - if gc_ll_descr.get_funcptr_for_newarray is not None: - ll_new_array = gc_ll_descr.get_funcptr_for_newarray() - self.malloc_array_func_addr = rffi.cast(lltype.Signed, - ll_new_array) - if gc_ll_descr.get_funcptr_for_newstr is not None: - ll_new_str = gc_ll_descr.get_funcptr_for_newstr() - self.malloc_str_func_addr = rffi.cast(lltype.Signed, - ll_new_str) - if gc_ll_descr.get_funcptr_for_newunicode is not None: - ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() - self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, - ll_new_unicode) - self.memcpy_addr = self.cpu.cast_ptr_to_int(codebuf.memcpy_fn) - self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) - self._build_failure_recovery(False) - self._build_failure_recovery(True) - if self.cpu.supports_floats: - self._build_failure_recovery(False, withfloats=True) - self._build_failure_recovery(True, withfloats=True) - codebuf.ensure_sse2_floats() - self._build_float_constants() - if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): - self._build_malloc_fixedsize_slowpath() - s = os.environ.get('PYPYLOG') - if s: - if s.find(':') != -1: - s = s.split(':')[-1] - self.set_debug(True) - self._output_loop_log = s + ".count" - # Intialize here instead of __init__ to prevent - # pending_guard_tokens from being considered a prebuilt object, - # which sometimes causes memory leaks since the prebuilt list is - # still considered a GC root after we re-assign - # pending_guard_tokens in write_pending_failure_recoveries - self.pending_guard_tokens = [] + def setup_once(self): + # the address of the function called by 'new' + gc_ll_descr = self.cpu.gc_ll_descr + gc_ll_descr.initialize() + ll_new = gc_ll_descr.get_funcptr_for_new() + self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) + if gc_ll_descr.get_funcptr_for_newarray is not None: + ll_new_array = gc_ll_descr.get_funcptr_for_newarray() + self.malloc_array_func_addr = rffi.cast(lltype.Signed, + ll_new_array) + if gc_ll_descr.get_funcptr_for_newstr is not None: + ll_new_str = gc_ll_descr.get_funcptr_for_newstr() + self.malloc_str_func_addr = rffi.cast(lltype.Signed, + ll_new_str) + if gc_ll_descr.get_funcptr_for_newunicode is not None: + ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() + self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, + ll_new_unicode) + self.memcpy_addr = self.cpu.cast_ptr_to_int(support.memcpy_fn) + self._build_failure_recovery(False) + self._build_failure_recovery(True) + if self.cpu.supports_floats: + self._build_failure_recovery(False, withfloats=True) + self._build_failure_recovery(True, withfloats=True) + support.ensure_sse2_floats() + self._build_float_constants() + if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): + self._build_malloc_fixedsize_slowpath() + debug_start('jit-backend-counts') + self.set_debug(have_debug_prints()) + debug_stop('jit-backend-counts') + + def setup(self, looptoken): + assert self.memcpy_addr != 0, "setup_once() not called?" + self.pending_guard_tokens = [] + self.mc = codebuf.MachineCodeBlockWrapper() + if self.datablockwrapper is None: + allblocks = self.get_asmmemmgr_blocks(looptoken) + self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, + allblocks) + + def teardown(self): + self.pending_guard_tokens = None + self.mc = None + self.looppos = -1 + self.currently_compiling_loop = None def finish_once(self): if self._debug: - output_log = self._output_loop_log - assert output_log is not None - f = open_file_as_stream(output_log, "w") + debug_start('jit-backend-counts') for i in range(len(self.loop_run_counters)): - name, struct = self.loop_run_counters[i] - f.write(str(name) + ":" + str(struct.i) + "\n") - f.close() + struct = self.loop_run_counters[i] + debug_print(str(i) + ':' + str(struct.i)) + debug_stop('jit-backend-counts') def _build_float_constants(self): - # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment - addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw', - track_allocation=False) - if not we_are_translated(): - self._keepalive_malloced_float_consts = addr - float_constants = rffi.cast(lltype.Signed, addr) - float_constants = (float_constants + 15) & ~15 # align to 16 bytes + datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, []) + float_constants = datablockwrapper.malloc_aligned(32, alignment=16) + datablockwrapper.done() addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' # 0x8000000000000000 @@ -271,45 +167,56 @@ def _build_malloc_fixedsize_slowpath(self): # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = self.mc.tell() + mc = codebuf.MachineCodeBlockWrapper() if self.cpu.supports_floats: # save the XMM registers in for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 - self.mc.MOVSD_sx((WORD*2)+8*i, i) - self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + mc.MOVSD_sx((WORD*2)+8*i, i) + mc.SUB_rr(edx.value, eax.value) # compute the size we want if IS_X86_32: - self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + mc.MOV_sr(WORD, edx.value) # save it as the new argument elif IS_X86_64: # rdi can be clobbered: its content was forced to the stack # by _fastpath_malloc(), like all other save_around_call_regs. - self.mc.MOV_rr(edi.value, edx.value) + mc.MOV_rr(edi.value, edx.value) addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - self.mc.JMP(imm(addr)) # tail call to the real malloc + mc.JMP(imm(addr)) # tail call to the real malloc + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.malloc_fixedsize_slowpath1 = rawstart # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = self.mc.tell() + mc = codebuf.MachineCodeBlockWrapper() if self.cpu.supports_floats: # restore the XMM registers for i in range(self.cpu.NUM_REGS):# from where they were saved - self.mc.MOVSD_xs(i, (WORD*2)+8*i) + mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - self.mc.RET() - self.mc.done() + mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + mc.RET() + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.malloc_fixedsize_slowpath2 = rawstart def assemble_loop(self, inputargs, operations, looptoken, log): - """adds the following attributes to looptoken: + '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) _x86_bootstrap_code (an integer giving an address) - _x86_direct_bootstrap_code + _x86_direct_bootstrap_code ( " " " " ) _x86_frame_depth _x86_param_depth _x86_arglocs _x86_debug_checksum - """ + ''' + # XXX this function is too longish and contains some code + # duplication with assemble_bridge(). Also, we should think + # about not storing on 'self' attributes that will live only + # for the duration of compiling one loop or a one bridge. + + clt = CompiledLoopToken(self.cpu, looptoken.number) + looptoken.compiled_loop_token = clt if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.setup() + self.setup(looptoken) + self.currently_compiling_loop = looptoken funcname = self._find_debug_merge_point(operations) if log: self._register_counter() @@ -319,43 +226,61 @@ arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - # profile support - name = "Loop # %s: %s" % (looptoken.number, funcname) - self.mc.start_function(name) - looptoken._x86_bootstrap_code = self.mc.tell() - adr_stackadjust = self._assemble_bootstrap_code(inputargs, arglocs) - curadr = self.mc.tell() - looptoken._x86_loop_code = curadr + bootstrappos = self.mc.get_relative_pos() + stackadjustpos = self._assemble_bootstrap_code(inputargs, arglocs) + self.looppos = self.mc.get_relative_pos() looptoken._x86_frame_depth = -1 # temporarily looptoken._x86_param_depth = -1 # temporarily frame_depth, param_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - looptoken._x86_direct_bootstrap_code = self.mc.tell() - self._assemble_bootstrap_direct_call(arglocs, curadr, + directbootstrappos = self.mc.get_relative_pos() + self._assemble_bootstrap_direct_call(arglocs, self.looppos, frame_depth+param_depth) - # - debug_print("Loop #%d has address %x to %x" % (looptoken.number, - looptoken._x86_loop_code, - self.mc.tell())) - self.mc.end_function() self.write_pending_failure_recoveries() - - def assemble_bridge(self, faildescr, inputargs, operations, log): + fullsize = self.mc.get_relative_pos() + # + rawstart = self.materialize_loop(looptoken) + debug_print("Loop #%d (%s) has address %x to %x" % ( + looptoken.number, funcname, + rawstart + self.looppos, + rawstart + directbootstrappos)) + self._patch_stackadjust(rawstart + stackadjustpos, + frame_depth + param_depth) + self.patch_pending_failure_recoveries(rawstart) + # + looptoken._x86_bootstrap_code = rawstart + bootstrappos + looptoken._x86_loop_code = rawstart + self.looppos + looptoken._x86_direct_bootstrap_code = rawstart + directbootstrappos + self.teardown() + # oprofile support + if self.cpu.profile_agent is not None: + name = "Loop # %s: %s" % (looptoken.number, funcname) + self.cpu.profile_agent.native_code_written(name, + rawstart, fullsize) + + def assemble_bridge(self, faildescr, inputargs, operations, + original_loop_token, log): if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.setup() + descr_number = self.cpu.get_fail_descr_number(faildescr) + try: + failure_recovery = self._find_failure_recovery_bytecode(faildescr) + except ValueError: + debug_print("Bridge out of guard", descr_number, + "was already compiled!") + return + + self.setup(original_loop_token) funcname = self._find_debug_merge_point(operations) if log: self._register_counter() operations = self._inject_debugging_code(faildescr, operations) - arglocs = self.rebuild_faillocs_from_descr( - faildescr._x86_failure_recovery_bytecode) + arglocs = self.rebuild_faillocs_from_descr(failure_recovery) if not we_are_translated(): assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) @@ -364,37 +289,63 @@ regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) - # oprofile support - descr_number = self.cpu.get_fail_descr_number(faildescr) - name = "Bridge # %s: %s" % (descr_number, funcname) - self.mc.start_function(name) - - adr_bridge = self.mc.tell() - adr_stackadjust = self._patchable_stackadjust() + stackadjustpos = self._patchable_stackadjust() frame_depth, param_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) + codeendpos = self.mc.get_relative_pos() + self.write_pending_failure_recoveries() + fullsize = self.mc.get_relative_pos() + # + rawstart = self.materialize_loop(original_loop_token) + + debug_print("Bridge out of guard %d (%s) has address %x to %x" % + (descr_number, funcname, rawstart, rawstart + codeendpos)) + self._patch_stackadjust(rawstart + stackadjustpos, + frame_depth + param_depth) + self.patch_pending_failure_recoveries(rawstart) if not we_are_translated(): # for the benefit of tests faildescr._x86_bridge_frame_depth = frame_depth faildescr._x86_bridge_param_depth = param_depth # patch the jump from original guard - self.patch_jump_for_descr(faildescr, adr_bridge) - debug_print("Bridge out of guard %d has address %x to %x" % - (descr_number, adr_bridge, self.mc.tell())) - self.mc.end_function() - self.write_pending_failure_recoveries() + self.patch_jump_for_descr(faildescr, rawstart) + self.teardown() + # oprofile support + if self.cpu.profile_agent is not None: + name = "Bridge # %s: %s" % (descr_number, funcname) + self.cpu.profile_agent.native_code_written(name, + rawstart, fullsize) def write_pending_failure_recoveries(self): + # for each pending guard, generate the code of the recovery stub + # at the end of self.mc. for tok in self.pending_guard_tokens: - # Okay to write to _mc because we've already made sure that - # there's enough space by "reserving" bytes. - addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) - tok.faildescr._x86_adr_recovery_stub = addr - self.patch_jump_for_descr(tok.faildescr, addr) + tok.pos_recovery_stub = self.generate_quick_failure(tok) - self.pending_guard_tokens = [] - self.mc.reset_reserved_bytes() - self.mc.done() + def patch_pending_failure_recoveries(self, rawstart): + # after we wrote the assembler to raw memory, set up + # tok.faildescr._x86_adr_jump_offset to contain the raw address of + # the 4-byte target field in the JMP/Jcond instruction, and patch + # the field in question to point (initially) to the recovery stub + for tok in self.pending_guard_tokens: + addr = rawstart + tok.pos_jump_offset + tok.faildescr._x86_adr_jump_offset = addr + relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4) + assert rx86.fits_in_32bits(relative_target) + p = rffi.cast(rffi.INTP, addr) + p[0] = rffi.cast(rffi.INT, relative_target) + + def get_asmmemmgr_blocks(self, looptoken): + clt = looptoken.compiled_loop_token + if clt.asmmemmgr_blocks is None: + clt.asmmemmgr_blocks = [] + return clt.asmmemmgr_blocks + + def materialize_loop(self, looptoken): + self.datablockwrapper.done() # finish using cpu.asmmemmgr + self.datablockwrapper = None + allblocks = self.get_asmmemmgr_blocks(looptoken) + return self.mc.materialize(self.cpu.asmmemmgr, allblocks, + self.cpu.gc_ll_descr.gcrootmap) def _find_debug_merge_point(self, operations): @@ -409,29 +360,50 @@ def _register_counter(self): if self._debug: + # YYY very minor leak -- we need the counters to stay alive + # forever, just because we want to report them at the end + # of the process struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', - track_allocation=False) # known to leak + track_allocation=False) struct.i = 0 - self.loop_run_counters.append((len(self.loop_run_counters), struct)) - + self.loop_run_counters.append(struct) + + def _find_failure_recovery_bytecode(self, faildescr): + adr_jump_offset = faildescr._x86_adr_jump_offset + if adr_jump_offset == 0: + raise ValueError + # follow the JMP/Jcond + p = rffi.cast(rffi.INTP, adr_jump_offset) + adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) + # skip the CALL + if WORD == 4: + adr_target += 5 # CALL imm + else: + adr_target += 13 # MOV r11, imm; CALL *r11 + return adr_target + def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - adr_recovery_stub = faildescr._x86_adr_recovery_stub + assert adr_jump_offset != 0 offset = adr_new_target - (adr_jump_offset + 4) # If the new target fits within a rel32 of the jump, just patch # that. Otherwise, leave the original rel32 to the recovery stub in # place, but clobber the recovery stub with a jump to the real # target. + mc = codebuf.MachineCodeBlockWrapper() if rx86.fits_in_32bits(offset): - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) mc.writeimm32(offset) + mc.copy_to_raw_memory(adr_jump_offset) else: - # "mov r11, addr; jmp r11" is 13 bytes - mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + # "mov r11, addr; jmp r11" is 13 bytes, which fits in there + # because we always write "mov r11, addr; call *r11" in the + # first place. mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) mc.JMP_r(X86_64_SCRATCH_REG.value) - - mc.done() + p = rffi.cast(rffi.INTP, adr_jump_offset) + adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) + mc.copy_to_raw_memory(adr_target) + faildescr._x86_adr_jump_offset = 0 # means "patched" @specialize.argtype(1) def _inject_debugging_code(self, looptoken, operations): @@ -442,7 +414,7 @@ s += op.getopnum() looptoken._x86_debug_checksum = s c_adr = ConstInt(rffi.cast(lltype.Signed, - self.loop_run_counters[-1][1])) + self.loop_run_counters[-1])) box = BoxInt() box2 = BoxInt() ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], @@ -451,19 +423,11 @@ ResOperation(rop.SETFIELD_RAW, [c_adr, box2], None, descr=self.debug_counter_descr)] operations = ops + operations - # # we need one register free (a bit of a hack, but whatever) - # self.mc.PUSH(eax) - # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) - # self.mc.MOV(eax, heap(adr)) - # self.mc.ADD(eax, imm1) - # self.mc.MOV(heap(adr), eax) - # self.mc.POP(eax) return operations def _assemble(self, regalloc, operations): self._regalloc = regalloc regalloc.walk_operations(operations) - self.mc.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -479,30 +443,31 @@ def _patchable_stackadjust(self): # stack adjustment LEA self.mc.LEA32_rb(esp.value, 0) - return self.mc.tell() - 4 + return self.mc.get_relative_pos() - 4 - def _patch_stackadjust(self, adr_lea, reserved_depth): + def _patch_stackadjust(self, adr_lea, allocated_depth): # patch stack adjustment LEA - mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) - # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. + mc = codebuf.MachineCodeBlockWrapper() + # Compute the correct offset for the instruction LEA ESP, [EBP-4*words] + mc.writeimm32(self._get_offset_of_ebp_from_esp(allocated_depth)) + mc.copy_to_raw_memory(adr_lea) + + def _get_offset_of_ebp_from_esp(self, allocated_depth): # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth - # align, e.g. for Mac OS X + words = (self.cpu.FRAME_FIXED_SIZE - 1) + allocated_depth + # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.writeimm32(-WORD * aligned_words) - mc.done() + return -WORD * aligned_words def _call_header(self): + # NB. the shape of the frame is hard-coded in get_basic_shape() too. + # Also, make sure this is consistent with FRAME_FIXED_SIZE. self.mc.PUSH_r(ebp.value) self.mc.MOV_rr(ebp.value, esp.value) for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) - # NB. the shape of the frame is hard-coded in get_basic_shape() too. - # Also, make sure this is consistent with FRAME_FIXED_SIZE. - return self._patchable_stackadjust() - def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -512,17 +477,16 @@ self.mc.POP_r(ebp.value) self.mc.RET() - def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + def _assemble_bootstrap_direct_call(self, arglocs, jmppos, stackdepth): if IS_X86_64: - return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) + return self._assemble_bootstrap_direct_call_64(arglocs, jmppos, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - # XXX not to repeat the logic, a bit around - adr_stackadjust = self._call_header() - self._patch_stackadjust(adr_stackadjust, stackdepth) + self._call_header() + self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if isinstance(loc, RegLoc): @@ -544,9 +508,11 @@ self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) assert isinstance(loc, StackLoc) self.mc.MOVSD_bx(loc.value, xmmtmp.value) - self.mc.JMP_l(jmpadr) + endpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(jmppos - endpos) + assert endpos == self.mc.get_relative_pos() - def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + def _assemble_bootstrap_direct_call_64(self, arglocs, jmppos, stackdepth): # XXX: Very similar to _emit_call_64 src_locs = [] @@ -560,8 +526,8 @@ unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - adr_stackadjust = self._call_header() - self._patch_stackadjust(adr_stackadjust, stackdepth) + self._call_header() + self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones assert len(nonfloatlocs) == len(floatlocs) @@ -597,8 +563,9 @@ # clobber the scratch register self.mc.MOV(loc, X86_64_SCRATCH_REG) - finaljmp = self.mc.tell() - self.mc.JMP(imm(jmpadr)) + endpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(jmppos - endpos) + assert endpos == self.mc.get_relative_pos() def redirect_call_assembler(self, oldlooptoken, newlooptoken): # some minimal sanity checking @@ -611,16 +578,17 @@ # Ideally we should rather patch all existing CALLs, but well. oldadr = oldlooptoken._x86_direct_bootstrap_code target = newlooptoken._x86_direct_bootstrap_code - mc = codebuf.InMemoryCodeBuilder(oldadr, oldadr + 16) + mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) - mc.done() + mc.copy_to_raw_memory(oldadr) def _assemble_bootstrap_code(self, inputargs, arglocs): nonfloatlocs, floatlocs = arglocs - adr_stackadjust = self._call_header() + self._call_header() + stackadjustpos = self._patchable_stackadjust() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] - self.mc._mc.begin_reuse_scratch_register() + self.mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: @@ -652,8 +620,8 @@ self.mc.MOVSD(xmmtmp, heap(adr)) assert isinstance(loc, StackLoc) self.mc.MOVSD_bx(loc.value, xmmtmp.value) - self.mc._mc.end_reuse_scratch_register() - return adr_stackadjust + self.mc.end_reuse_scratch_register() + return stackadjustpos def dump(self, text): if not self.verbose: @@ -661,7 +629,8 @@ _prev = Box._extended_display try: Box._extended_display = False - print >> sys.stderr, ' 0x%x %s' % (fixid(self.mc.tell()), text) + pos = self.mc.get_relative_pos() + print >> sys.stderr, ' 0x%x %s' % (pos, text) finally: Box._extended_display = _prev @@ -721,7 +690,7 @@ arglocs, resloc) if not we_are_translated(): # must be added by the genop_guard_list[]() - assert hasattr(faildescr, '_x86_adr_jump_offset') + assert guard_token is self.pending_guard_tokens[-1] def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc, current_depths): @@ -794,9 +763,6 @@ result_loc): guard_opnum = guard_op.getopnum() self.mc.UCOMISD(arglocs[0], arglocs[1]) - # 16 is enough space for the rel8 jumps below and the rel32 - # jump in implement_guard - self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: if need_jp: self.mc.J_il8(rx86.Conditions['P'], 6) @@ -964,9 +930,6 @@ def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.getopnum() self.mc.UCOMISD(arglocs[0], arglocs[1]) - # 16 is enough space for the rel8 jumps below and the rel32 - # jump in implement_guard - self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: self.mc.J_il8(rx86.Conditions['P'], 6) self.implement_guard(guard_token, 'E') @@ -1290,13 +1253,11 @@ self.mc.CMP32_mi((locs[0].value, 0), expected_typeid) def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): - self.mc.ensure_bytes_available(256) self._cmp_guard_class(locs) self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, guard_token, locs, ign_2): - self.mc.ensure_bytes_available(256) self.mc.CMP(locs[0], imm1) # Patched below self.mc.J_il8(rx86.Conditions['B'], 0) @@ -1305,7 +1266,7 @@ # patch the JB above offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, [chr(offset)]) + self.mc.overwrite(jb_location-1, chr(offset)) # self.implement_guard(guard_token, 'NE') @@ -1314,42 +1275,33 @@ exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - desc_bytes = self.failure_recovery_description(failargs, fail_locs) - return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) + return GuardToken(faildescr, failargs, fail_locs, exc) - def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): + def generate_quick_failure(self, guardtok): """Generate the initial code for handling a failure. We try to - keep it as compact as possible. The idea is that this code is - executed at most once (and very often, zero times); when - executed, it generates a more complete piece of code which can - really handle recovery from this particular failure. + keep it as compact as possible. """ - fail_index = self.cpu.get_fail_descr_number(faildescr) - addr = mc.tell() + fail_index = self.cpu.get_fail_descr_number(guardtok.faildescr) + mc = self.mc + startpos = mc.get_relative_pos() withfloats = False - for box in failargs: + for box in guardtok.failargs: if box is not None and box.type == FLOAT: withfloats = True break + exc = guardtok.exc mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery - faildescr._x86_failure_recovery_bytecode = mc.tell() - for byte in desc_bytes: - mc.writechr(ord(byte)) + self.write_failure_recovery_description(mc, guardtok.failargs, + guardtok.fail_locs) # write the fail_index too mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): - mc.writechr(0xCC) - faildescr._x86_debug_faillocs = [loc for loc in fail_locs - if loc is not None] - - # Make sure the recovery stub is at least 16 bytes long (for the - # case where we overwrite the recovery stub with a 64-bit absolute - # jump) - while mc.tell() - addr < 16: - mc.writechr(0x00) - return addr + mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.fail_locs if loc is not None] + guardtok.faildescr._x86_debug_faillocs = faillocs + return startpos DESCR_REF = 0x00 DESCR_INT = 0x01 @@ -1360,8 +1312,7 @@ CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def failure_recovery_description(self, failargs, locs): - desc_bytes = [] + def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1381,19 +1332,14 @@ n = loc.value n = kind + 4*n while n > 0x7F: - desc_bytes.append(chr((n & 0x7F) | 0x80)) + mc.writechar(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - desc_bytes.append(chr(n)) - desc_bytes.append(chr(self.CODE_STOP)) + mc.writechar(chr(n)) + mc.writechar(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE - return desc_bytes - - def write_failure_recovery_description(self, mc, failargs, locs): - for byte in self.failure_recovery_description(failargs, locs): - mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager @@ -1531,10 +1477,8 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc._mc - # Assume that we are called at the beginning, when there is no risk - # that 'mc' runs out of space. Checked by asserts in mc.write(). - recovery_addr = mc.tell() + mc = codebuf.MachineCodeBlockWrapper() + self.mc = mc # Push all general purpose registers for gpr in range(self.cpu.NUM_REGS-1, -1, -1): @@ -1585,11 +1529,12 @@ # above. self._call_footer() - self.mc.done() - self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None def generate_failure(self, fail_index, locs, exc, locs_are_ref): - self.mc._mc.begin_reuse_scratch_register() + self.mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] if isinstance(loc, RegLoc): @@ -1616,7 +1561,7 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(eax, loc) self.mc.MOV(heap(adr), eax) - self.mc._mc.end_reuse_scratch_register() + self.mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1632,17 +1577,13 @@ self._call_footer() def implement_guard(self, guard_token, condition=None): - self.mc.reserve_bytes(guard_token.recovery_stub_size()) - self.pending_guard_tokens.append(guard_token) - # These jumps are patched later, the mc.tell() are just - # dummy values. Also, use self.mc._mc to avoid triggering a - # "buffer full" exactly here. - mc = self.mc._mc + # These jumps are patched later. if condition: - mc.J_il(rx86.Conditions[condition], mc.tell()) + self.mc.J_il(rx86.Conditions[condition], 0) else: - mc.JMP_l(mc.tell()) - guard_token.faildescr._x86_adr_jump_offset = mc.tell() - 4 + self.mc.JMP_l(0) + guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4 + self.pending_guard_tokens.append(guard_token) def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] @@ -1697,7 +1638,6 @@ # Write a call to the direct_bootstrap_code of the target assembler self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1733,7 +1673,7 @@ # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - self.mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, chr(offset)) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1768,7 +1708,7 @@ # Here we join Path A and Path B again offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.overwrite(jmp_location - 1, chr(offset)) self.mc.CMP_bi(FORCE_INDEX_OFS, 0) self.implement_guard(guard_token, 'L') @@ -1783,9 +1723,6 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - # ensure that enough bytes are available to write the whole - # following piece of code atomically (for the JZ) - self.mc.ensure_bytes_available(256) self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), descr.jit_wb_if_flag_singlebyte) self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later @@ -1828,7 +1765,7 @@ # patch the JZ above offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - self.mc.overwrite(jz_location-1, [chr(offset)]) + self.mc.overwrite(jz_location-1, chr(offset)) def genop_force_token(self, op, arglocs, resloc): # RegAlloc.consider_force_token ensures this: @@ -1851,18 +1788,21 @@ gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: mark = self._regalloc.get_mark_gc_roots(gcrootmap) - gcrootmap.put(rffi.cast(llmemory.Address, self.mc.tell()), mark) + self.mc.insert_gcroot_marker(mark) def target_arglocs(self, loop_token): return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(imm(loop_token._x86_loop_code)) + if loop_token is self.currently_compiling_loop: + curpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(self.looppos - curpos) + else: + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): size = max(size, self.cpu.gc_ll_descr.minimal_size_in_nursery) - self.mc.ensure_bytes_available(256) self.mc.MOV(eax, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (eax.value, size)) self.mc.CMP(edx, heap(nursery_top_adr)) @@ -1890,7 +1830,7 @@ offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.overwrite(jmp_adr-1, chr(offset)) # on 64-bits, 'tid' is a value that fits in 31 bits self.mc.MOV_mi((eax.value, 0), tid) self.mc.MOV(heap(nursery_free_adr), edx) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/regalloc.py Thu Dec 9 16:33:59 2010 @@ -60,32 +60,6 @@ r15: 5, } -class FloatConstants(object): - BASE_CONSTANT_SIZE = 1000 - - def __init__(self): - self.cur_array_free = 0 - self.const_id = 0 - - def _get_new_array(self): - n = self.BASE_CONSTANT_SIZE - # known to leak - self.cur_array = lltype.malloc(rffi.CArray(lltype.Float), n, - flavor='raw', track_allocation=False) - self.cur_array_free = n - _get_new_array._dont_inline_ = True - - def record_float(self, floatval): - if self.cur_array_free == 0: - self._get_new_array() - arr = self.cur_array - n = self.cur_array_free - 1 - arr[n] = floatval - self.cur_array_free = n - self.const_id += 1 - return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) - - class X86XMMRegisterManager(RegisterManager): box_types = [FLOAT] @@ -93,20 +67,11 @@ # we never need lower byte I hope save_around_call_regs = all_regs - def __init__(self, longevity, frame_manager=None, assembler=None): - RegisterManager.__init__(self, longevity, frame_manager=frame_manager, - assembler=assembler) - if assembler is None: - self.float_constants = FloatConstants() - else: - if assembler._float_constants is None: - assembler._float_constants = FloatConstants() - self.float_constants = assembler._float_constants - def convert_to_imm(self, c): - const_id, adr = self.float_constants.record_float(c.getfloat()) - return ConstFloatLoc(adr, const_id) - + adr = self.assembler.datablockwrapper.malloc_aligned(8, 8) + rffi.cast(rffi.CArrayPtr(rffi.DOUBLE), adr)[0] = c.getfloat() + return ConstFloatLoc(adr) + def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately @@ -320,11 +285,22 @@ def locs_for_fail(self, guard_op): return [self.loc(v) for v in guard_op.getfailargs()] + def get_current_depth(self): + # return (self.fm.frame_depth, self.param_depth), but trying to share + # the resulting tuple among several calls + arg0 = self.fm.frame_depth + arg1 = self.param_depth + result = self.assembler._current_depths_cache + if result[0] != arg0 or result[1] != arg1: + result = (arg0, arg1) + self.assembler._current_depths_cache = result + return result + def perform_with_guard(self, op, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) self.rm.position += 1 self.xrm.position += 1 - current_depths = (self.fm.frame_depth, self.param_depth) + current_depths = self.get_current_depth() self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, current_depths) @@ -340,7 +316,7 @@ arglocs)) else: self.assembler.dump('%s(%s)' % (guard_op, arglocs)) - current_depths = (self.fm.frame_depth, self.param_depth) + current_depths = self.get_current_depth() self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, current_depths) @@ -1097,7 +1073,8 @@ if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert reg in self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX gcrootmap.add_callee_save_reg(shape, self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX[reg]) - return gcrootmap.compress_callshape(shape) + return gcrootmap.compress_callshape(shape, + self.assembler.datablockwrapper) def consider_force_token(self, op): loc = self.rm.force_allocate_reg(op.result) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/regloc.py Thu Dec 9 16:33:59 2010 @@ -177,24 +177,15 @@ class ConstFloatLoc(AssemblerLocation): # XXX: We have to use this class instead of just AddressLoc because - # AddressLoc is "untyped" and also we to have need some sort of unique - # identifier that we can use in _getregkey (for jump.py) - + # we want a width of 8 (... I think. Check this!) _immutable_ = True - width = 8 - def __init__(self, address, const_id): + def __init__(self, address): self.value = address - self.const_id = const_id def __repr__(self): - return '' % (self.value, self.const_id) - - def _getregkey(self): - # XXX: 1000 is kind of magic: We just don't want to be confused - # with any registers - return 1000 + self.const_id + return '' % (self.value,) def location_code(self): return 'j' @@ -334,9 +325,9 @@ if code == possible_code: val = getattr(loc, "value_" + possible_code)() if possible_code == 'i': - offset = intmask(val - (self.tell() + 5)) - if rx86.fits_in_32bits(offset): + if self.WORD == 4: _rx86_getattr(self, name + "_l")(val) + self.add_pending_relocation() else: assert self.WORD == 8 self._load_scratch(val) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/runner.py Thu Dec 9 16:33:59 2010 @@ -44,6 +44,7 @@ def setup_once(self): self.profile_agent.startup() + self.assembler.setup_once() def finish_once(self): self.assembler.finish_once() @@ -53,9 +54,12 @@ self.assembler.assemble_loop(inputargs, operations, looptoken, log=log) - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): + clt = original_loop_token.compiled_loop_token + clt.compiling_a_bridge() self.assembler.assemble_bridge(faildescr, inputargs, operations, - log=log) + original_loop_token, log=log) def set_future_value_int(self, index, intvalue): self.assembler.fail_boxes_int.setitem(index, intvalue) @@ -124,8 +128,8 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = -1 - bytecode = rffi.cast(rffi.UCHARP, - faildescr._x86_failure_recovery_bytecode) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) # start of "no gc operation!" block fail_index_2 = self.assembler.grab_frame_values( bytecode, @@ -165,17 +169,12 @@ CPU = CPU386 # silence warnings - -history.LoopToken._x86_param_depth = 0 -history.LoopToken._x86_arglocs = (None, None) -history.LoopToken._x86_frame_depth = 0 -history.LoopToken._x86_bootstrap_code = 0 -history.LoopToken._x86_direct_bootstrap_code = 0 -history.LoopToken._x86_failure_recovery_bytecode = 0 -history.LoopToken._x86_loop_code = 0 -history.LoopToken._x86_current_depths = (0, 0) - -compile._DoneWithThisFrameDescr._x86_current_depths = (0, 0) -compile._DoneWithThisFrameDescr._x86_failure_recovery_bytecode = 0 -compile._DoneWithThisFrameDescr._x86_adr_jump_offset = 0 - +##history.LoopToken._x86_param_depth = 0 +##history.LoopToken._x86_arglocs = (None, None) +##history.LoopToken._x86_frame_depth = 0 +##history.LoopToken._x86_bootstrap_code = 0 +##history.LoopToken._x86_direct_bootstrap_code = 0 +##history.LoopToken._x86_loop_code = 0 +##history.LoopToken._x86_debug_checksum = 0 +##compile.AbstractFailDescr._x86_current_depths = (0, 0) +##compile.AbstractFailDescr._x86_adr_jump_offset = 0 Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/rx86.py Thu Dec 9 16:33:59 2010 @@ -137,10 +137,9 @@ # ____________________________________________________________ # Emit an immediate displacement (relative to the cur insn) -def encode_relative(mc, target, _, orbyte): +def encode_relative(mc, relative_target, _, orbyte): assert orbyte == 0 - offset = intmask(target - (mc.tell() + 4)) - mc.writeimm32(offset) + mc.writeimm32(relative_target) return 0 def relative(argnum): Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/support.py Thu Dec 9 16:33:59 2010 @@ -1,4 +1,7 @@ +import sys from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from pypy.translator.tool.cbuild import ExternalCompilationInfo + def values_array(TP, size): ATP = lltype.GcArray(TP) @@ -23,3 +26,22 @@ return True return ValuesArray() + +# ____________________________________________________________ + +memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) + +# ____________________________________________________________ + +if sys.platform == 'win32': + ensure_sse2_floats = lambda : None +else: + _sse2_eci = ExternalCompilationInfo( + compile_extra = ['-msse2', '-mfpmath=sse'], + separate_module_sources = ['void PYPY_NO_OP(void) {}'], + ) + ensure_sse2_floats = rffi.llexternal('PYPY_NO_OP', [], lltype.Void, + compilation_info=_sse2_eci, + sandboxsafe=True) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_assembler.py Thu Dec 9 16:33:59 2010 @@ -1,5 +1,5 @@ from pypy.jit.backend.x86.regloc import * -from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper +from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask @@ -19,28 +19,10 @@ return 42 class FakeMC: - def __init__(self, base_address=0): + def __init__(self): self.content = [] - self._size = 100 - self.base_address = base_address - def writechr(self, n): - self.content.append(n) - def tell(self): - return self.base_address + len(self.content) - def get_relative_pos(self): - return len(self.content) - def JMP(self, *args): - self.content.append(("JMP", args)) - def done(self): - pass - def PUSH_r(self, reg): - pass - def POP_r(self, reg): - pass - -class FakeAssembler: - def write_pending_failure_recoveries(self): - pass + def writechar(self, char): + self.content.append(ord(char)) def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -255,41 +237,3 @@ assert assembler.fail_boxes_int.getitem(i) == expected_ints[i] assert assembler.fail_boxes_ptr.getitem(i) == expected_ptrs[i] assert assembler.fail_boxes_float.getitem(i) == expected_floats[i] - -class FakeProfileAgent(object): - def __init__(self): - self.functions = [] - - def native_code_written(self, name, address, size): - self.functions.append((name, address, size)) - -class FakeMCWrapper(MachineCodeBlockWrapper): - count = 0 - def _instantiate_mc(self): - self.count += 1 - return FakeMC(200 * (self.count - 1)) - -def test_mc_wrapper_profile_agent(): - agent = FakeProfileAgent() - assembler = FakeAssembler() - mc = FakeMCWrapper(assembler, 100, agent) - mc.start_function("abc") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4)] - mc.writechr("x") - mc.start_function("cde") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4), ("cde", 5, 4)] - mc.start_function("xyz") - for i in range(50): - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4), ("cde", 5, 4), ("xyz", 9, 29), ("xyz", 200, 22)] Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_gc_integration.py Thu Dec 9 16:33:59 2010 @@ -33,7 +33,8 @@ def add_callee_save_reg(self, shape, reg_index): index_to_name = { 1: 'ebx', 2: 'esi', 3: 'edi' } shape.append(index_to_name[reg_index]) - def compress_callshape(self, shape): + def compress_callshape(self, shape, datablockwrapper): + assert datablockwrapper == 'fakedatablockwrapper' assert shape[0] == 'shape' return ['compressed'] + shape[1:] @@ -58,7 +59,9 @@ def test_mark_gc_roots(self): cpu = CPU(None, None) + cpu.setup_once() regalloc = RegAlloc(MockAssembler(cpu, MockGcDescr(False))) + regalloc.assembler.datablockwrapper = 'fakedatablockwrapper' boxes = [BoxPtr() for i in range(len(X86RegisterManager.all_regs))] longevity = {} for box in boxes: @@ -90,6 +93,7 @@ cpu = CPU(None, None) cpu.gc_ll_descr = MockGcDescr(False) + cpu.setup_once() S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', ('field', lltype.Ptr(S)), @@ -214,6 +218,7 @@ cpu = CPU(None, None) cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() + cpu.setup_once() NODE = lltype.Struct('node', ('tid', lltype.Signed), ('value', lltype.Signed)) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regalloc.py Thu Dec 9 16:33:59 2010 @@ -9,7 +9,7 @@ from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ - FloatConstants, is_comparison_or_ovf_op + is_comparison_or_ovf_op from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -41,7 +41,10 @@ self.movs = [] self.performs = [] self.lea = [] - self.cpu = cpu or CPU(None, None) + if cpu is None: + cpu = CPU(None, None) + cpu.setup_once() + self.cpu = cpu if gc_ll_descr is None: gc_ll_descr = MockGcDescr(False) self.cpu.gc_ll_descr = gc_ll_descr @@ -76,6 +79,7 @@ class BaseTestRegalloc(object): cpu = CPU(None, None) + cpu.setup_once() def raising_func(i): if i: @@ -166,7 +170,8 @@ assert ([box.type for box in bridge.inputargs] == [box.type for box in guard_op.getfailargs()]) faildescr = guard_op.getdescr() - self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations) + self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations, + loop.token) return bridge def run(self, loop): @@ -515,16 +520,6 @@ self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] - def test_float_overflow_const_list(self): - ops = ['[f0]'] - BASE_CONSTANT_SIZE = FloatConstants.BASE_CONSTANT_SIZE - for i in range(BASE_CONSTANT_SIZE * 2): - ops.append('f%d = float_add(f%d, 3.5)' % (i + 1, i)) - ops.append('finish(f%d)' % (BASE_CONSTANT_SIZE * 2)) - ops = "\n".join(ops) - self.interpret(ops, [0.1]) - assert abs(self.getfloat(0) - (BASE_CONSTANT_SIZE * 2) * 3.5 - 0.1) < 0.00001 - def test_lt_const(self): ops = ''' [f0] Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regalloc2.py Thu Dec 9 16:33:59 2010 @@ -19,6 +19,7 @@ ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, 9) @@ -41,6 +42,7 @@ ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, -10) @@ -137,6 +139,7 @@ ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, -13) @@ -251,6 +254,7 @@ ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, 17) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_regloc.py Thu Dec 9 16:33:59 2010 @@ -1,4 +1,4 @@ -import struct +import struct, sys from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.test.test_rx86 import CodeBuilder32, CodeBuilder64, assert_encodes_as from pypy.jit.backend.x86.assembler import heap @@ -37,26 +37,36 @@ assert_encodes_as(cb64, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') assert_encodes_as(cb64, "CMP16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\x81\x7D\x00\x39\x30') -def test_jmp_wraparound(): - if not IS_X86_32: - py.test.skip() - - pos_addr = intmask(0x7FFFFF00) - neg_addr = intmask(0x800000BB) - - # JMP to "negative" address from "positive" address - s = cb32() - s.base_address = pos_addr - s.JMP(ImmedLoc(neg_addr)) - expected_ofs = neg_addr - (pos_addr+5) - assert s.getvalue() == '\xE9' + struct.pack(" sys.maxint: + continue + mc = codebuf.MachineCodeBlockWrapper() + mc.CALL(ImmedLoc(target)) + length = mc.get_relative_pos() + buf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw') + rawstart = rffi.cast(lltype.Signed, buf) + if IS_X86_32: + assert length == 5 + assert mc.relocations == [5] + expected = "\xE8" + struct.pack('= 40 # randomish number @@ -370,7 +376,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" # Would be exactly ==, but there are some guard failure recovery @@ -397,104 +403,15 @@ assert res.value == 4.0 -class TestX86OverflowMC(TestX86): - - def setup_method(self, meth): - self.cpu = CPU(rtyper=None, stats=FakeStats()) - self.cpu.assembler.mc_size = 1024 - - def test_overflow_mc(self): - ops = [] - base_v = BoxInt() - v = base_v - for i in range(1024): - next_v = BoxInt() - ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) - v = next_v - ops.append(ResOperation(rop.FINISH, [v], None, - descr=BasicFailDescr())) - looptoken = LoopToken() - self.cpu.assembler.setup() - old_mc_mc = self.cpu.assembler.mc._mc - self.cpu.compile_loop([base_v], ops, looptoken) - assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed - self.cpu.set_future_value_int(0, base_v.value) - self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 1024 - - def test_overflow_guard_float_cmp(self): - # The float comparisons on x86 tend to use small relative jumps, - # which may run into trouble if they fall on the edge of a - # MachineCodeBlock change. - a = BoxFloat(1.0) - b = BoxFloat(2.0) - failed = BoxInt(41) - finished = BoxInt(42) - - # We select guards that will always succeed, so that execution will - # continue through the entire set of comparisions - ops_to_test = ( - (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), - - (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), - - (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), - - (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), - - (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), - (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), - - (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), - (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), - ) - - for float_op, args, guard_op in ops_to_test: - ops = [] - - for i in range(200): - cmp_result = BoxInt() - ops.append(ResOperation(float_op, args, cmp_result)) - ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) - ops[-1].setfailargs([failed]) - - ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) - - looptoken = LoopToken() - self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) - self.cpu.set_future_value_float(0, a.value) - self.cpu.set_future_value_float(1, b.value) - self.cpu.set_future_value_int(2, failed.value) - self.cpu.set_future_value_int(3, finished.value) - self.cpu.execute_token(looptoken) - - # Really just a sanity check. We're actually interested in - # whether the test segfaults. - assert self.cpu.get_latest_value_int(0) == finished.value - - def test_overflow_guard_exception(self): - for i in range(50): - self.test_exceptions() - - class TestDebuggingAssembler(object): def setup_method(self, meth): - self.pypylog = os.environ.get('PYPYLOG', None) - self.logfile = str(udir.join('x86_runner.log')) - os.environ['PYPYLOG'] = "mumble:" + self.logfile self.cpu = CPU(rtyper=None, stats=FakeStats()) - - def teardown_method(self, meth): - if self.pypylog is not None: - os.environ['PYPYLOG'] = self.pypylog + self.cpu.setup_once() def test_debugger_on(self): + from pypy.tool.logparser import parse_log_file, extract_category + from pypy.rlib import debug + loop = """ [i0] debug_merge_point('xyz', 0) @@ -504,17 +421,19 @@ jump(i1) """ ops = parse(loop) - self.cpu.assembler.set_debug(True) - self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) - self.cpu.set_future_value_int(0, 0) - self.cpu.execute_token(ops.token) - # check debugging info - name, struct = self.cpu.assembler.loop_run_counters[0] - assert name == 0 # 'xyz' - assert struct.i == 10 - self.cpu.finish_once() - lines = py.path.local(self.logfile + ".count").readlines() - assert lines[0] == '0:10\n' # '10 xyz\n' + debug._log = dlog = debug.DebugLog() + try: + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + struct = self.cpu.assembler.loop_run_counters[0] + assert struct.i == 10 + self.cpu.finish_once() + finally: + debug._log = None + assert ('jit-backend-counts', [('debug_print', '0:10')]) in dlog def test_debugger_checksum(self): loop = """ Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/x86/test/test_rx86.py Thu Dec 9 16:33:59 2010 @@ -5,7 +5,6 @@ class CodeBuilderMixin(object): def __init__(self): self.buffer = [] - self.base_address = 0x76543210 def writechar(self, c): assert isinstance(c, str) and len(c) == 1 @@ -14,9 +13,6 @@ def getvalue(self): return ''.join(self.buffer) - def tell(self): - return self.base_address + len(self.buffer) - def assert_encodes_as(code_builder_cls, insn_name, args, expected_encoding): s = code_builder_cls() getattr(s, insn_name)(*args) @@ -104,21 +100,18 @@ def test_call_l(s=None): s = s or CodeBuilder32() - s.CALL_l(0x01234567) - ofs = 0x01234567 - (0x76543210+5) - assert s.getvalue() == '\xE8' + struct.pack(" 5: + x = A1 + else: + x = A2 + x() + rtyper = support.annotate(f, [35]) + maingraph = rtyper.annotator.translator.graphs[0] + cw = CodeWriter(FakeCPU(rtyper), [FakeJitDriverSD(maingraph)]) + cw.find_all_graphs(MyFakePolicy()) + cw.make_jitcodes(verbose=True) + # + names = [jitcode.name for jitcode in cw.assembler.indirectcalltargets] + assert len(names) == 1 + assert names[0].startswith('instantiate_') and names[0].endswith('A1') + # + print cw.assembler.list_of_addr2name + names = dict.fromkeys([value + for key, value in cw.assembler.list_of_addr2name]) + assert 'A1' in names + assert 'A2' in names + def test_int_abs(): def f(n): return abs(n) Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_list.py Thu Dec 9 16:33:59 2010 @@ -56,9 +56,8 @@ if '/' in oopspec_name: oopspec_name, property = oopspec_name.split('/') def force_flags(op): - if property == 'NONNEG': return True, False - if property == 'NEG': return False, False - if property == 'CANRAISE': return False, True + if property == 'NONNEG': return True + if property == 'NEG': return False raise ValueError(property) tr._get_list_nonneg_canraise_flags = force_flags op = SpaceOperation('direct_call', @@ -122,9 +121,6 @@ check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_i %r0, , %i1 -> %i2 """) - builtin_test('list.getitem/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -139,9 +135,6 @@ check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_pure_i %r0, , %i1 -> %i2 """) - builtin_test('list.getitem_foldable/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -158,10 +151,6 @@ check_neg_index %r0, , %i0 -> %i1 setarrayitem_gc_i %r0, , %i1, %i2 """) - builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), - varoftype(lltype.Signed), - varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -206,9 +195,6 @@ check_resizable_neg_index %r0, , %i0 -> %i1 getlistitem_gc_i %r0, , , %i1 -> %i2 """) - builtin_test('list.getitem/CANRAISE', - [varoftype(VARLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_resizable_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), @@ -225,10 +211,6 @@ check_resizable_neg_index %r0, , %i0 -> %i1 setlistitem_gc_i %r0, , , %i1, %i2 """) - builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), - varoftype(lltype.Signed), - varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_resizable_len(): builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_regalloc.py Thu Dec 9 16:33:59 2010 @@ -281,22 +281,22 @@ # this used to produce bogus code, containing these two # lines in the following broken order: # last_exc_value -> %r0 - # ref_copy %r0 -> %r2 -- but expect to read the old value of %r0! + # ref_copy %r0 -> %r1 -- but expect to read the old value of %r0! self.check_assembler(graph, """ residual_call_r_r $<* fn bar>, , R[%r0] -> %r1 -live- - residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r2 + residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r1 -live- catch_exception L1 - ref_return %r2 + ref_return %r1 --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 - ref_copy %r0 -> %r2 + ref_copy %r0 -> %r1 last_exc_value -> %r0 residual_call_r_r $<* fn foo>, , R[%r0] -> %r0 -live- - ref_return %r2 + ref_return %r1 --- L2: reraise Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_void_list.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_void_list.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_void_list.py Thu Dec 9 16:33:59 2010 @@ -51,9 +51,6 @@ builtin_test('list.getitem/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -62,9 +59,6 @@ builtin_test('list.getitem_foldable/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem_foldable/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -75,10 +69,6 @@ varoftype(lltype.Signed), varoftype(lltype.Void)], lltype.Void, "") - builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), - varoftype(lltype.Signed), - varoftype(lltype.Void)], - lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -115,9 +105,6 @@ builtin_test('list.getitem/NEG', [varoftype(VARLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem/CANRAISE', - [varoftype(VARLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_resizable_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), @@ -128,10 +115,6 @@ varoftype(lltype.Signed), varoftype(lltype.Void)], lltype.Void, "") - builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), - varoftype(lltype.Signed), - varoftype(lltype.Void)], - lltype.Void, NotSupported) def test_resizable_len(): builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, Modified: pypy/branch/out-of-line-guards/pypy/jit/conftest.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/conftest.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/conftest.py Thu Dec 9 16:33:59 2010 @@ -1,7 +1,5 @@ import py -option = py.test.config.option - def pytest_addoption(parser): group = parser.getgroup("JIT options") group.addoption('--slow', action="store_true", Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py Thu Dec 9 16:33:59 2010 @@ -1,4 +1,4 @@ - +import weakref from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype from pypy.objspace.flow.model import Constant, Variable @@ -48,6 +48,32 @@ loop_token.outermost_jitdriver_sd = jitdriver_sd return loop_token +def record_loop_or_bridge(loop): + """Do post-backend recordings and cleanups on 'loop'. + """ + # get the original loop token (corresponding to 'loop', or if that is + # a bridge, to the loop that this bridge belongs to) + looptoken = loop.token + assert looptoken is not None + wref = weakref.ref(looptoken) + for op in loop.operations: + descr = op.getdescr() + if isinstance(descr, ResumeDescr): + descr.wref_original_loop_token = wref # stick it there + n = descr.index + if n >= 0: # we also record the resumedescr number + looptoken.compiled_loop_token.record_faildescr_index(n) + elif isinstance(descr, LoopToken): + # for a JUMP or a CALL_ASSEMBLER: record it as a potential jump. + # (the following test is not enough to prevent more complicated + # cases of cycles, but at least it helps in simple tests of + # test_memgr.py) + if descr is not looptoken: + looptoken.record_jump_to(descr) + op.setdescr(None) # clear reference, mostly for tests + # mostly for tests: make sure we don't keep a reference to the LoopToken + loop.token = None + # ____________________________________________________________ def compile_new_loop(metainterp, old_loop_tokens, start): @@ -77,6 +103,7 @@ return old_loop_token send_loop_to_backend(metainterp_sd, loop, "loop") insert_loop_token(old_loop_tokens, loop_token) + record_loop_or_bridge(loop) return loop_token def insert_loop_token(old_loop_tokens, loop_token): @@ -117,8 +144,11 @@ else: loop._ignore_during_counting = True metainterp_sd.log("compiled new " + type) + if metainterp_sd.warmrunnerdesc is not None: # for tests + metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(loop.token) -def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): +def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations, + original_loop_token): n = metainterp_sd.cpu.get_fail_descr_number(faildescr) metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) if not we_are_translated(): @@ -127,7 +157,8 @@ metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: - metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, + original_loop_token) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() @@ -227,9 +258,6 @@ CNT_FLOAT = -0x60000000 CNT_MASK = 0x1FFFFFFF - def __init__(self, metainterp_sd): - self.metainterp_sd = metainterp_sd - def store_final_boxes(self, guard_op, boxes): guard_op.setfailargs(boxes) self.guard_opnum = guard_op.getopnum() @@ -314,12 +342,14 @@ def compile_and_attach(self, metainterp, new_loop): # We managed to create a bridge. Attach the new operations - # to the corrsponding guard_op and compile from there + # to the corresponding guard_op and compile from there + assert metainterp.resumekey_original_loop_token is not None + new_loop.token = metainterp.resumekey_original_loop_token inputargs = metainterp.history.inputargs if not we_are_translated(): self._debug_suboperations = new_loop.operations send_bridge_to_backend(metainterp.staticdata, self, inputargs, - new_loop.operations) + new_loop.operations, new_loop.token) def copy_all_attrbutes_into(self, res): # XXX a bit ugly to have to list them all here @@ -331,14 +361,14 @@ res.rd_pendingfields = self.rd_pendingfields def _clone_if_mutable(self): - res = ResumeGuardDescr(self.metainterp_sd) + res = ResumeGuardDescr() self.copy_all_attrbutes_into(res) return res class ResumeGuardForcedDescr(ResumeGuardDescr): def __init__(self, metainterp_sd, jitdriver_sd): - ResumeGuardDescr.__init__(self, metainterp_sd) + self.metainterp_sd = metainterp_sd self.jitdriver_sd = jitdriver_sd def handle_fail(self, metainterp_sd, jitdriver_sd): @@ -482,6 +512,8 @@ metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd redargs = new_loop.inputargs + # We make a new LoopToken for this entry bridge, and stick it + # to every guard in the loop. new_loop_token = make_loop_token(len(redargs), jitdriver_sd) new_loop.token = new_loop_token send_loop_to_backend(metainterp_sd, new_loop, "entry bridge") @@ -489,12 +521,14 @@ jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( self.original_greenkey, new_loop_token) - # store the new loop in compiled_merge_points too + # store the new loop in compiled_merge_points_wref too old_loop_tokens = metainterp.get_compiled_merge_points( self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token old_loop_tokens.append(new_loop_token) + metainterp.set_compiled_merge_points(self.original_greenkey, + old_loop_tokens) def reset_counter_from_failure(self): pass @@ -529,6 +563,7 @@ # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr) prepare_last_operation(new_loop, target_loop_token) resumekey.compile_and_attach(metainterp, new_loop) + record_loop_or_bridge(new_loop) return target_loop_token def prepare_last_operation(new_loop, target_loop_token): @@ -555,7 +590,8 @@ propagate_exception_descr = PropagateExceptionDescr() -def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redboxes): +def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redboxes, + memory_manager=None): """Make a LoopToken that corresponds to assembler code that just calls back the interpreter. Used temporarily: a fully compiled version of the code may end up replacing it. @@ -595,4 +631,6 @@ ] operations[1].setfailargs([]) cpu.compile_loop(inputargs, operations, loop_token, log=False) + if memory_manager is not None: # for tests + memory_manager.keep_loop_alive(loop_token) return loop_token Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py Thu Dec 9 16:33:59 2010 @@ -4,8 +4,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic from pypy.rlib.objectmodel import compute_hash, compute_unique_id -from pypy.rlib.rarithmetic import intmask -from pypy.tool.uid import uid +from pypy.rlib.rarithmetic import intmask, r_int64 from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -730,14 +729,28 @@ outermost_jitdriver_sd = None # specnodes = ... # and more data specified by the backend when the loop is compiled - number = 0 + number = -1 + generation = r_int64(0) + # one purpose of LoopToken is to keep alive the CompiledLoopToken + # returned by the backend. When the LoopToken goes away, the + # CompiledLoopToken has its __del__ called, which frees the assembler + # memory and the ResumeGuards. + compiled_loop_token = None - def __init__(self, number=0): - self.number = number + def __init__(self): + # For memory management of assembled loops + self._keepalive_target_looktokens = {} # set of other LoopTokens + + def record_jump_to(self, target_loop_token): + self._keepalive_target_looktokens[target_loop_token] = None + + def __repr__(self): + return '' % (self.number, self.generation) def repr_of_descr(self): return '' % self.number + class TreeLoop(object): inputargs = None operations = None Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/jitprof.py Thu Dec 9 16:33:59 2010 @@ -2,9 +2,9 @@ """ A small helper module for profiling JIT """ -import os import time -from pypy.rlib.debug import debug_print +from pypy.rlib.debug import debug_print, debug_start, debug_stop +from pypy.rlib.debug import have_debug_prints from pypy.jit.metainterp.jitexc import JitException counters=""" @@ -24,6 +24,10 @@ NVIRTUALS NVHOLES NVREUSED +TOTAL_COMPILED_LOOPS +TOTAL_COMPILED_BRIDGES +TOTAL_FREED_LOOPS +TOTAL_FREED_BRIDGES """ def _setup(): @@ -35,6 +39,7 @@ _setup() JITPROF_LINES = ncounters + 1 + 1 # one for TOTAL, 1 for calls, update if needed +_CPU_LINES = 4 # the last 4 lines are stored on the cpu class BaseProfiler(object): pass @@ -48,9 +53,6 @@ def finish(self): pass - def set_printing(self, printing): - pass - def start_tracing(self): pass @@ -90,23 +92,19 @@ counters = None calls = 0 current = None - printing = True + cpu = None def start(self): self.starttime = self.timer() self.t1 = self.starttime self.times = [0, 0] - self.counters = [0] * ncounters + self.counters = [0] * (ncounters - _CPU_LINES) self.calls = 0 self.current = [] def finish(self): self.tk = self.timer() - if self.printing: - self.print_stats() - - def set_printing(self, printing): - self.printing = printing + self.print_stats() def _start(self, event): t0 = self.t1 @@ -154,6 +152,12 @@ self.calls += 1 def print_stats(self): + debug_start("jit-summary") + if have_debug_prints(): + self._print_stats() + debug_stop("jit-summary") + + def _print_stats(self): cnt = self.counters tim = self.times calls = self.calls @@ -161,8 +165,8 @@ self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) self._print_intline("Running asm", cnt[RUNNING]) self._print_intline("Blackhole", cnt[BLACKHOLE]) - line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) - os.write(2, line) + line = "TOTAL: \t\t%f" % (self.tk - self.starttime, ) + debug_print(line) self._print_intline("ops", cnt[OPS]) self._print_intline("recorded ops", cnt[RECORDED_OPS]) self._print_intline(" calls", calls) @@ -176,17 +180,26 @@ self._print_intline("nvirtuals", cnt[NVIRTUALS]) self._print_intline("nvholes", cnt[NVHOLES]) self._print_intline("nvreused", cnt[NVREUSED]) + cpu = self.cpu + if cpu is not None: # for some tests + self._print_intline("Total # of loops", + cpu.total_compiled_loops) + self._print_intline("Total # of bridges", + cpu.total_compiled_bridges) + self._print_intline("Freed # of loops", + cpu.total_freed_loops) + self._print_intline("Freed # of bridges", + cpu.total_freed_bridges) def _print_line_time(self, string, i, tim): - final = "%s:%s\t%d\t%f\n" % (string, " " * max(0, 13-len(string)), i, tim) - os.write(2, final) + final = "%s:%s\t%d\t%f" % (string, " " * max(0, 13-len(string)), i, tim) + debug_print(final) def _print_intline(self, string, i): final = string + ':' + " " * max(0, 16-len(string)) - final += '\t' + str(i) + '\n' - os.write(2, final) - - + final += '\t' + str(i) + debug_print(final) + class BrokenProfilerData(JitException): pass Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizefindnode.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizefindnode.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizefindnode.py Thu Dec 9 16:33:59 2010 @@ -128,6 +128,7 @@ assert isinstance(constbox, Const) node = InstanceNode() node.unique = UNIQUE_NO + node.escaped = True node.knownvaluebox = constbox self.nodes[box] = node return node Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/__init__.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/__init__.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/__init__.py Thu Dec 9 16:33:59 2010 @@ -3,7 +3,6 @@ from pypy.jit.metainterp.optimizeopt.intbounds import OptIntBounds from pypy.jit.metainterp.optimizeopt.virtualize import OptVirtualize from pypy.jit.metainterp.optimizeopt.heap import OptHeap -from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall from pypy.jit.metainterp.optimizeopt.string import OptString def optimize_loop_1(metainterp_sd, loop, virtuals=True): @@ -17,6 +16,10 @@ OptVirtualize(), OptString(), OptHeap(), + ] + if metainterp_sd.jit_ffi: + from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall + optimizations = optimizations + [ OptFfiCall(), ] optimizer = Optimizer(metainterp_sd, loop, optimizations, virtuals) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/optimizer.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/optimizer.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/optimizeopt/optimizer.py Thu Dec 9 16:33:59 2010 @@ -301,7 +301,9 @@ return CVAL_ZERO def propagate_all_forward(self): - self.exception_might_have_happened = False + self.exception_might_have_happened = True + # ^^^ at least at the start of bridges. For loops, we could set + # it to False, but we probably don't care self.newoperations = [] self.i = 0 while self.i < len(self.loop.operations): Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py Thu Dec 9 16:33:59 2010 @@ -14,11 +14,10 @@ from pypy.jit.metainterp.logger import Logger from pypy.jit.metainterp.jitprof import EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE -from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG +from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE from pypy.jit.metainterp.jitexc import JitException, get_llexception from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize -from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr from pypy.jit.codewriter import heaptracker @@ -94,6 +93,18 @@ else: raise AssertionError(argcode) outvalue[startindex+i] = reg + def _put_back_list_of_boxes(self, outvalue, startindex, position): + code = self.bytecode + length = ord(code[position]) + position += 1 + for i in range(length): + index = ord(code[position+i]) + box = outvalue[startindex+i] + if box.type == history.INT: self.registers_i[index] = box + elif box.type == history.REF: self.registers_r[index] = box + elif box.type == history.FLOAT: self.registers_f[index] = box + else: raise AssertionError(box.type) + def get_current_position_info(self): return self.jitcode.get_live_vars_info(self.pc) @@ -601,8 +612,10 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) arrayindex = vinfo.array_field_by_descrs[arrayfielddescr] index = indexbox.getint() - if index < 0: - index += vinfo.get_array_length(virtualizable, arrayindex) + # Support for negative index: disabled + # (see codewriter/jtransform.py, _check_no_vable_array). + #if index < 0: + # index += vinfo.get_array_length(virtualizable, arrayindex) assert 0 <= index < vinfo.get_array_length(virtualizable, arrayindex) return vinfo.get_index_in_array(virtualizable, arrayindex, index) @@ -814,8 +827,9 @@ for i in range(num_green_args): assert isinstance(varargs[i], Const) - @arguments("orgpc", "int", "boxes3", "boxes3") - def opimpl_jit_merge_point(self, orgpc, jdindex, greenboxes, redboxes): + @arguments("orgpc", "int", "boxes3", "jitcode_position", "boxes3") + def opimpl_jit_merge_point(self, orgpc, jdindex, greenboxes, + jcposition, redboxes): any_operation = len(self.metainterp.history.operations) > 0 jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] self.verify_green_args(jitdriver_sd, greenboxes) @@ -843,6 +857,10 @@ self.pc = orgpc self.metainterp.reached_loop_header(greenboxes, redboxes) self.pc = saved_pc + # no exception, which means that the jit_merge_point did not + # close the loop. We have to put the possibly-modified list + # 'redboxes' back into the registers where it comes from. + put_back_list_of_boxes3(self, jcposition, redboxes) else: # warning! careful here. We have to return from the current # frame containing the jit_merge_point, and then use @@ -1052,7 +1070,7 @@ resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, metainterp.jitdriver_sd) else: - resumedescr = compile.ResumeGuardDescr(metainterp_sd) + resumedescr = compile.ResumeGuardDescr() guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None @@ -1198,7 +1216,8 @@ logger_ops = None def __init__(self, cpu, options, - ProfilerClass=EmptyProfiler, warmrunnerdesc=None): + ProfilerClass=EmptyProfiler, warmrunnerdesc=None, + jit_ffi=True): self.cpu = cpu self.stats = self.cpu.stats self.options = options @@ -1206,7 +1225,9 @@ self.logger_ops = Logger(self, guard_number=True) self.profiler = ProfilerClass() + self.profiler.cpu = cpu self.warmrunnerdesc = warmrunnerdesc + self.jit_ffi = jit_ffi backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] @@ -1324,6 +1345,11 @@ return jitcode return None + def try_to_free_some_loops(self): + # Increase here the generation recorded by the memory manager. + if self.warmrunnerdesc is not None: # for tests + self.warmrunnerdesc.memory_manager.next_generation() + # ---------------- logging ------------------------ def log(self, msg): @@ -1620,6 +1646,7 @@ self.staticdata._setup_once() self.staticdata.profiler.start_tracing() assert jitdriver_sd is self.jitdriver_sd + self.staticdata.try_to_free_some_loops() self.create_empty_history() try: original_boxes = self.initialize_original_boxes(jitdriver_sd, *args) @@ -1648,10 +1675,15 @@ debug_start('jit-tracing') self.staticdata.profiler.start_tracing() assert isinstance(key, compile.ResumeGuardDescr) + # store the resumekey.wref_original_loop_token() on 'self' to make + # sure that it stays alive as long as this MetaInterp + self.resumekey_original_loop_token = key.wref_original_loop_token() + self.staticdata.try_to_free_some_loops() self.initialize_state_from_guard_failure(key) try: return self._handle_guard_failure(key) finally: + self.resumekey_original_loop_token = None self.staticdata.profiler.end_tracing() debug_stop('jit-tracing') @@ -1661,6 +1693,8 @@ self.seen_loop_header_for_jdindex = -1 try: self.prepare_resume_from_failure(key.guard_opnum) + if self.resumekey_original_loop_token is None: # very rare case + raise SwitchToBlackhole(ABORT_BRIDGE) self.interpret() except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) @@ -1782,10 +1816,15 @@ raise NotImplementedError(opname[opnum]) def get_compiled_merge_points(self, greenkey): + """Get the list of looptokens corresponding to the greenkey. + Turns the (internal) list of weakrefs into regular refs. + """ cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) - if cell.compiled_merge_points is None: - cell.compiled_merge_points = [] - return cell.compiled_merge_points + return cell.get_compiled_merge_points() + + def set_compiled_merge_points(self, greenkey, looptokens): + cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) + cell.set_compiled_merge_points(looptokens) def compile(self, original_boxes, live_arg_boxes, start): num_green_args = self.jitdriver_sd.num_green_args @@ -1795,6 +1834,7 @@ self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None) loop_token = compile.compile_new_loop(self, old_loop_tokens, start) if loop_token is not None: # raise if it *worked* correctly + self.set_compiled_merge_points(greenkey, old_loop_tokens) raise GenerateMergePoint(live_arg_boxes, loop_token) self.history.operations.pop() # remove the JUMP @@ -2293,6 +2333,8 @@ else: raise AssertionError("bad argcode") position += 1 + elif argtype == "jitcode_position": + value = position else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) @@ -2337,3 +2379,15 @@ argtypes = unrolling_iterable(unboundmethod.argtypes) handler.func_name = 'handler_' + name return handler + +def put_back_list_of_boxes3(frame, position, newvalue): + code = frame.bytecode + length1 = ord(code[position]) + position2 = position + 1 + length1 + length2 = ord(code[position2]) + position3 = position2 + 1 + length2 + length3 = ord(code[position3]) + assert len(newvalue) == length1 + length2 + length3 + frame._put_back_list_of_boxes(newvalue, 0, position) + frame._put_back_list_of_boxes(newvalue, length1, position2) + frame._put_back_list_of_boxes(newvalue, length1 + length2, position3) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py Thu Dec 9 16:33:59 2010 @@ -19,7 +19,11 @@ from pypy.jit.metainterp import simple_optimize class FakeJitCell: - compiled_merge_points = None + __compiled_merge_points = [] + def get_compiled_merge_points(self): + return self.__compiled_merge_points[:] + def set_compiled_merge_points(self, lst): + self.__compiled_merge_points = lst class FakeWarmRunnerState: def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): @@ -35,7 +39,6 @@ optimize_bridge = staticmethod(simple_optimize.optimize_bridge) trace_limit = sys.maxint - debug_level = 2 func._jit_unroll_safe_ = True rtyper = support.annotate(func, values, type_system=type_system) @@ -1716,6 +1719,29 @@ res = self.meta_interp(f, [5, 2]) assert 4 < res < 14 + def test_compute_identity_hash(self): + from pypy.rlib.objectmodel import compute_identity_hash + class A(object): + pass + def f(): + a = A() + return compute_identity_hash(a) == compute_identity_hash(a) + res = self.interp_operations(f, []) + assert res + # a "did not crash" kind of test + + def test_compute_unique_id(self): + from pypy.rlib.objectmodel import compute_unique_id + class A(object): + pass + def f(): + a1 = A() + a2 = A() + return (compute_unique_id(a1) == compute_unique_id(a1) and + compute_unique_id(a1) != compute_unique_id(a2)) + res = self.interp_operations(f, []) + assert res + class TestOOtype(BasicTests, OOJitMixin): Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_compile.py Thu Dec 9 16:33:59 2010 @@ -42,7 +42,6 @@ class FakeState: optimize_loop = staticmethod(optimize.optimize_loop) - debug_level = 0 class FakeGlobalData: loopnumbering = 0 @@ -54,6 +53,8 @@ stats = Stats() profiler = jitprof.EmptyProfiler() + warmrunnerdesc = None + jit_ffi = False def log(self, msg, event_kind=None): pass @@ -207,14 +208,12 @@ class ExitFrameWithExceptionRef(Exception): pass FakeMetaInterpSD.cpu = cpu - class FakeJitDriverSD: - pass cpu.set_future_value_int(0, -156) cpu.set_future_value_int(1, -178) cpu.set_future_value_int(2, -190) fail_descr = cpu.execute_token(loop_token) try: - fail_descr.handle_fail(FakeMetaInterpSD(), FakeJitDriverSD()) + fail_descr.handle_fail(FakeMetaInterpSD(), None) except FakeMetaInterpSD.ExitFrameWithExceptionRef, e: assert lltype.cast_opaque_ptr(lltype.Ptr(EXC), e.args[1]) == llexc else: Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_exception.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_exception.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_exception.py Thu Dec 9 16:33:59 2010 @@ -1,6 +1,6 @@ import py, sys from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE +from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE, dont_look_inside from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask from pypy.jit.codewriter.policy import StopAtXPolicy @@ -588,6 +588,33 @@ res = self.interp_operations(f, [99]) assert res == 21 + def test_bug_exc1_noexc_exc2(self): + myjitdriver = JitDriver(greens=[], reds=['i']) + @dont_look_inside + def rescall(i): + if i < 10: + raise KeyError + if i < 20: + return None + raise ValueError + def f(i): + while i < 30: + myjitdriver.can_enter_jit(i=i) + myjitdriver.jit_merge_point(i=i) + try: + rescall(i) + except KeyError: + assert i < 10 + except ValueError: + assert i >= 20 + else: + assert 10 <= i < 20 + i += 1 + return i + res = self.meta_interp(f, [0], inline=True) + assert res == 30 + + class MyError(Exception): def __init__(self, n): self.n = n Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_fficall.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_fficall.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_fficall.py Thu Dec 9 16:33:59 2010 @@ -40,6 +40,6 @@ n += 1 return res # - res = self.meta_interp(f, [0]) + res = self.meta_interp(f, [0], jit_ffi=True) return res Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_list.py Thu Dec 9 16:33:59 2010 @@ -188,6 +188,26 @@ assert res == f(4) self.check_loops(call=0, getfield_gc=0) + def test_fold_indexerror(self): + jitdriver = JitDriver(greens = [], reds = ['total', 'n', 'lst']) + def f(n): + lst = [] + total = 0 + while n > 0: + jitdriver.can_enter_jit(lst=lst, n=n, total=total) + jitdriver.jit_merge_point(lst=lst, n=n, total=total) + lst.append(n) + try: + total += lst[n] + except IndexError: + total += 1000 + n -= 1 + return total + + res = self.meta_interp(f, [15], listops=True) + assert res == f(15) + self.check_loops(guard_exception=0) + class TestOOtype(ListTests, OOJitMixin): pass Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_logger.py Thu Dec 9 16:33:59 2010 @@ -14,11 +14,20 @@ def capturing(func, *args, **kwds): log_stream = StringIO() - debug._stderr = log_stream + class MyDebugLog: + def debug_print(self, *args): + for arg in args: + print >> log_stream, arg, + print >> log_stream + def debug_start(self, *args): + pass + def debug_stop(self, *args): + pass try: + debug._log = MyDebugLog() func(*args, **kwds) finally: - debug._stderr = sys.stderr + debug._log = None return log_stream.getvalue() class Logger(logger.Logger): @@ -112,7 +121,8 @@ equaloplists(loop.operations, oloop.operations) def test_jump(self): - namespace = {'target': LoopToken(3)} + namespace = {'target': LoopToken()} + namespace['target'].number = 3 inp = ''' [i0] jump(i0, descr=target) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizefficall.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizefficall.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizefficall.py Thu Dec 9 16:33:59 2010 @@ -31,6 +31,7 @@ class TestFfiCall(BaseTestOptimizeOpt, LLtypeMixin): + jit_ffi = True class namespace: cpu = LLtypeMixin.cpu Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_optimizeopt.py Thu Dec 9 16:33:59 2010 @@ -28,11 +28,12 @@ class FakeMetaInterpStaticData(object): - def __init__(self, cpu): + def __init__(self, cpu, jit_ffi=False): self.cpu = cpu self.profiler = EmptyProfiler() self.options = Fake() self.globaldata = Fake() + self.jit_ffi = jit_ffi def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr @@ -41,7 +42,7 @@ b1 = BoxInt() opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu), None) - fdescr = ResumeGuardDescr(None) + fdescr = ResumeGuardDescr() op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr) # setup rd data fi0 = resume.FrameInfo(None, "code0", 11) @@ -227,6 +228,7 @@ return sorted(boxes, key=lambda box: _kind2count[box.type]) class BaseTestOptimizeOpt(BaseTest): + jit_ffi = False def invent_fail_descr(self, fail_args): if fail_args is None: @@ -261,7 +263,7 @@ loop.token.specnodes = self.unpack_specnodes(spectext) # self.loop = loop - metainterp_sd = FakeMetaInterpStaticData(self.cpu) + metainterp_sd = FakeMetaInterpStaticData(self.cpu, self.jit_ffi) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo if hasattr(self, 'callinfocollection'): @@ -805,8 +807,12 @@ i3 = call(i2, descr=nonwritedescr) jump(i1) # the exception is considered lost when we loop back """ + # note that 'guard_no_exception' at the very start must be kept + # around: bridges may start with one. (In case of loops we could + # remove it, but we probably don't care.) expected = """ [i] + guard_no_exception() [] i1 = int_add(i, 3) i2 = call(i1, descr=nonwritedescr) guard_no_exception() [i1, i2] @@ -1966,6 +1972,15 @@ """ self.optimize_loop(ops, 'Virtual(node_vtable2, nextdescr=Not, otherdescr=Not)', expected) + def test_bug_4(self): + ops = """ + [p9] + p30 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) + jump(p30) + """ + self.optimize_loop(ops, 'Not', ops) + def test_invalid_loop_1(self): ops = """ [p1] Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_pyjitpl.py Thu Dec 9 16:33:59 2010 @@ -17,6 +17,7 @@ portal.setup(None) class FakeStaticData: cpu = None + warmrunnerdesc = None metainterp = pyjitpl.MetaInterp(FakeStaticData(), None) metainterp.framestack = [] @@ -53,6 +54,7 @@ def test_remove_consts_and_duplicates(): class FakeStaticData: cpu = None + warmrunnerdesc = None def is_another_box_like(box, referencebox): assert box is not referencebox assert isinstance(box, referencebox.clonebox().__class__) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_recursive.py Thu Dec 9 16:33:59 2010 @@ -927,12 +927,16 @@ x=x) frame.s = hint(frame.s, promote=True) n -= 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] frame.s += 1 if codeno == 0: subframe = Frame([n, n+1, n+2, n+3], 0) x += f(1, 10, 1, subframe) - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] x += len(frame.l) frame.s -= 1 return x @@ -1142,6 +1146,19 @@ res = self.meta_interp(main, [], inline=True, trace_limit=tlimit) assert ''.join(res.chars) == 'ABCDEFGHIabcdefghijJ' * 5 + def test_no_duplicates_bug(self): + driver = JitDriver(greens = ['codeno'], reds = ['i'], + get_printable_location = lambda codeno: str(codeno)) + def portal(codeno, i): + while i > 0: + driver.can_enter_jit(codeno=codeno, i=i) + driver.jit_merge_point(codeno=codeno, i=i) + if codeno > 0: + break + portal(i, i) + i -= 1 + self.meta_interp(portal, [0, 10], inline=True) + class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_virtualizable.py Thu Dec 9 16:33:59 2010 @@ -480,9 +480,13 @@ myjitdriver.jit_merge_point(frame=frame, n=n, x=x) frame.s = hint(frame.s, promote=True) n -= 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] frame.s += 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] x += len(frame.l) frame.s -= 1 return x @@ -994,7 +998,9 @@ jitdriver.can_enter_jit(frame=frame, n=n) jitdriver.jit_merge_point(frame=frame, n=n) popped = frame.stack[frame.stackpos] - frame.stackpos -= 1 + sp = frame.stackpos - 1 + assert sp >= 0 + frame.stackpos = sp to_push = intmask(popped * 3) frame.stack[frame.stackpos] = to_push frame.stackpos += 1 Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmspot.py Thu Dec 9 16:33:59 2010 @@ -134,91 +134,6 @@ assert state.optimize_loop is optimize.optimize_loop assert state.optimize_bridge is optimize.optimize_bridge - def test_static_debug_level(self, capfd): - py.test.skip("debug_level is being deprecated") - from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS - from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler - - myjitdriver = JitDriver(greens = [], reds = ['n']) - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - return n - - capfd.readouterr() - self.meta_interp(f, [10], debug_level=DEBUG_OFF, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not "Running asm" in err - self.meta_interp(f, [10], debug_level=DEBUG_PROFILE, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not 'compiled new' in err - assert "Running asm" in err - - self.meta_interp(f, [10], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert 'ENTER' in err - assert 'LEAVE' in err - assert "Running asm" in err - - self.meta_interp(f, [10], debug_level=DEBUG_STEPS, - ProfilerClass=EmptyProfiler) - out, err = capfd.readouterr() - assert 'ENTER' in err - assert 'LEAVE' in err - assert not "Running asm" in err - - def test_set_param_debug(self): - py.test.skip("debug_level is being deprecated") - from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS - from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler - - myjitdriver = JitDriver(greens = [], reds = ['n']) - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - return n - - def main(n, debug): - myjitdriver.set_param("debug", debug) - print f(n) - - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_OFF], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not "Running asm" in err - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_PROFILE], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not 'compiled new' in err - assert "Running asm" in err - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_STEPS], debug_level=DEBUG_OFF, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert 'ENTER' in err - assert 'LEAVE' in err - assert "Running asm" in err - def test_unwanted_loops(self): mydriver = JitDriver(reds = ['n', 'total', 'm'], greens = []) @@ -377,23 +292,30 @@ exc_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) cls.exc_vtable = exc_vtable - class FakeFailDescr(object): + class FakeLoopToken: def __init__(self, no): self.no = no + self.generation = 0 + + class FakeFailDescr(object): + def __init__(self, looptoken): + assert isinstance(looptoken, FakeLoopToken) + self.looptoken = looptoken def handle_fail(self, metainterp_sd, jitdrivers_sd): - if self.no == 0: + no = self.looptoken.no + if no == 0: raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3) - if self.no == 1: + if no == 1: raise metainterp_sd.warmrunnerdesc.ContinueRunningNormally( [0], [], [], [1], [], []) - if self.no == 3: + if no == 3: exc = lltype.malloc(OBJECT) exc.typeptr = exc_vtable raise metainterp_sd.warmrunnerdesc.ExitFrameWithExceptionRef( metainterp_sd.cpu, lltype.cast_opaque_ptr(llmemory.GCREF, exc)) - return self.no + return self.looptoken class FakeDescr: def as_vtable_size_descr(self): @@ -418,11 +340,11 @@ sizeof = nodescr def get_fail_descr_from_number(self, no): - return FakeFailDescr(no) + return FakeFailDescr(FakeLoopToken(no)) def execute_token(self, token): - assert token == 2 - return FakeFailDescr(1) + assert token.no == 2 + return FakeFailDescr(FakeLoopToken(1)) driver = JitDriver(reds = ['red'], greens = ['green']) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_warmstate.py Thu Dec 9 16:33:59 2010 @@ -99,6 +99,8 @@ lltype.Float], lltype.Void)) class FakeWarmRunnerDesc: rtyper = FakeRTyper() + cpu = None + memory_manager = None class FakeJitDriverSD: _get_jitcell_at_ptr = llhelper(GETTER, getter) _set_jitcell_at_ptr = llhelper(SETTER, setter) @@ -126,6 +128,7 @@ future_values[j] = "float", value class FakeWarmRunnerDesc: cpu = FakeCPU() + memory_manager = None class FakeJitDriverSD: _red_args_types = ["int", "float"] virtualizable_info = None @@ -154,16 +157,20 @@ _get_jitcell_at_ptr = None state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() + class FakeLoopToken(object): + pass + looptoken = FakeLoopToken() state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], - "entry loop token") + looptoken) cell1 = get_jitcell(True, 5, 2.25) assert cell1.counter < 0 - assert cell1.entry_loop_token == "entry loop token" + assert cell1.get_entry_loop_token() is looptoken def test_make_jitdriver_callbacks_1(): class FakeWarmRunnerDesc: cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None @@ -189,6 +196,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) @@ -211,6 +219,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None @@ -233,6 +242,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py Thu Dec 9 16:33:59 2010 @@ -12,11 +12,12 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.debug import debug_print, fatalerror +from pypy.rlib.debug import debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.unsimplify import call_final_function -from pypy.jit.metainterp import history, pyjitpl, gc +from pypy.jit.metainterp import history, pyjitpl, gc, memmgr from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler @@ -24,21 +25,17 @@ from pypy.jit.metainterp.jitdriver import JitDriverStaticData from pypy.jit.codewriter import support, codewriter from pypy.jit.codewriter.policy import JitPolicy -from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE # ____________________________________________________________ # Bootstrapping -def apply_jit(translator, backend_name="auto", debug_level=DEBUG_STEPS, - inline=False, - **kwds): +def apply_jit(translator, backend_name="auto", inline=False, **kwds): if 'CPUClass' not in kwds: from pypy.jit.backend.detect_cpu import getcpuclass kwds['CPUClass'] = getcpuclass(backend_name) - if debug_level > DEBUG_OFF: - ProfilerClass = Profiler - else: - ProfilerClass = EmptyProfiler + ProfilerClass = Profiler + # Always use Profiler here, which should have a very low impact. + # Otherwise you can try with ProfilerClass = EmptyProfiler. warmrunnerdesc = WarmRunnerDesc(translator, translate_support_code=True, listops=True, @@ -47,7 +44,6 @@ **kwds) for jd in warmrunnerdesc.jitdrivers_sd: jd.warmstate.set_param_inlining(inline) - jd.warmstate.set_param_debug(debug_level) warmrunnerdesc.finish() translator.warmrunnerdesc = warmrunnerdesc # for later debugging @@ -66,7 +62,7 @@ def jittify_and_run(interp, graph, args, repeat=1, backendopt=False, trace_limit=sys.maxint, - debug_level=DEBUG_STEPS, inline=False, **kwds): + inline=False, loop_longevity=0, **kwds): from pypy.config.config import ConfigError translator = interp.typer.annotator.translator try: @@ -83,7 +79,7 @@ jd.warmstate.set_param_trace_eagerness(2) # for tests jd.warmstate.set_param_trace_limit(trace_limit) jd.warmstate.set_param_inlining(inline) - jd.warmstate.set_param_debug(debug_level) + jd.warmstate.set_param_loop_longevity(loop_longevity) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -158,9 +154,11 @@ class WarmRunnerDesc(object): def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, - optimizer=None, ProfilerClass=EmptyProfiler, **kwds): + optimizer=None, ProfilerClass=EmptyProfiler, + jit_ffi=None, **kwds): pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) + self.memory_manager = memmgr.MemoryManager() self.build_cpu(CPUClass, **kwds) self.find_portals() self.codewriter = codewriter.CodeWriter(self.cpu, self.jitdrivers_sd) @@ -172,8 +170,10 @@ self.check_access_directly_sanity(graphs) if backendopt: self.prejit_optimizations(policy, graphs) + elif self.opt.listops: + self.prejit_optimizations_minimal_inline(policy, graphs) - self.build_meta_interp(ProfilerClass) + self.build_meta_interp(ProfilerClass, jit_ffi) self.make_args_specifications() # from pypy.jit.metainterp.virtualref import VirtualRefInfo @@ -270,6 +270,10 @@ remove_asserts=True, really_remove_asserts=True) + def prejit_optimizations_minimal_inline(self, policy, graphs): + from pypy.translator.backendopt.inline import auto_inline_graphs + auto_inline_graphs(self.translator, graphs, 0.01) + def build_cpu(self, CPUClass, translate_support_code=False, no_stats=False, **kwds): assert CPUClass is not None @@ -288,11 +292,14 @@ translate_support_code, gcdescr=self.gcdescr) self.cpu = cpu - def build_meta_interp(self, ProfilerClass): + def build_meta_interp(self, ProfilerClass, jit_ffi=None): + if jit_ffi is None: + jit_ffi = self.translator.config.translation.jit_ffi self.metainterp_sd = MetaInterpStaticData(self.cpu, self.opt, ProfilerClass=ProfilerClass, - warmrunnerdesc=self) + warmrunnerdesc=self, + jit_ffi=jit_ffi) def make_virtualizable_infos(self): vinfos = {} @@ -724,7 +731,7 @@ loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) except JitException, e: return handle_jitexception(e) - fail_descr = self.cpu.execute_token(loop_token) + fail_descr = self.execute_token(loop_token) jd._assembler_call_helper = assembler_call_helper # for debugging jd._assembler_helper_ptr = self.helper_func( @@ -823,3 +830,10 @@ py.test.skip("rewrite_force_virtual: port it to ootype") all_graphs = self.translator.graphs vrefinfo.replace_force_virtual_with_call(all_graphs) + + # ____________________________________________________________ + + def execute_token(self, loop_token): + fail_descr = self.cpu.execute_token(loop_token) + self.memory_manager.keep_loop_alive(loop_token) + return fail_descr Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmstate.py Thu Dec 9 16:33:59 2010 @@ -1,4 +1,4 @@ -import sys +import sys, weakref from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rffi from pypy.rpython.ootypesystem import ootype from pypy.rpython.annlowlevel import hlstr, llstr, cast_base_ptr_to_instance @@ -9,7 +9,6 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.jit import (PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL, OPTIMIZER_NO_PERFECTSPEC) -from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import history @@ -150,9 +149,34 @@ # counter == -1: there is an entry bridge for this cell # counter == -2: tracing is currently going on for this cell counter = 0 - compiled_merge_points = None + compiled_merge_points_wref = None # list of weakrefs to LoopToken dont_trace_here = False - entry_loop_token = None + wref_entry_loop_token = None # (possibly) one weakref to LoopToken + + def get_compiled_merge_points(self): + result = [] + if self.compiled_merge_points_wref is not None: + for wref in self.compiled_merge_points_wref: + looptoken = wref() + if looptoken is not None: + result.append(looptoken) + return result + + def set_compiled_merge_points(self, looptokens): + self.compiled_merge_points_wref = [self._makeref(token) + for token in looptokens] + + def get_entry_loop_token(self): + if self.wref_entry_loop_token is not None: + return self.wref_entry_loop_token() + return None + + def set_entry_loop_token(self, looptoken): + self.wref_entry_loop_token = self._makeref(looptoken) + + def _makeref(self, looptoken): + assert looptoken is not None + return weakref.ref(looptoken) # ____________________________________________________________ @@ -165,6 +189,8 @@ "NOT_RPYTHON" self.warmrunnerdesc = warmrunnerdesc self.jitdriver_sd = jitdriver_sd + if warmrunnerdesc is not None: # for tests + self.cpu = warmrunnerdesc.cpu try: self.profiler = warmrunnerdesc.metainterp_sd.profiler except AttributeError: # for tests @@ -176,7 +202,7 @@ meth(default_value) def set_param_threshold(self, threshold): - if threshold < 0: + if threshold <= 0: self.increment_threshold = 0 # never reach the THRESHOLD_LIMIT return if threshold < 2: @@ -209,10 +235,11 @@ else: raise ValueError("unknown optimizer") - def set_param_debug(self, value): - self.debug_level = value - if self.profiler is not None: - self.profiler.set_printing(value >= DEBUG_PROFILE) + def set_param_loop_longevity(self, value): + # note: it's a global parameter, not a per-jitdriver one + if (self.warmrunnerdesc is not None and + self.warmrunnerdesc.memory_manager is not None): # all for tests + self.warmrunnerdesc.memory_manager.set_max_age(value) def disable_noninlinable_function(self, greenkey): cell = self.jit_cell_at_key(greenkey) @@ -225,12 +252,15 @@ def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): cell = self.jit_cell_at_key(greenkey) - cell.counter = -1 - old_token = cell.entry_loop_token - cell.entry_loop_token = entry_loop_token + old_token = cell.get_entry_loop_token() + cell.set_entry_loop_token(entry_loop_token) + cell.counter = -1 # valid entry bridge attached if old_token is not None: - cpu = self.warmrunnerdesc.cpu - cpu.redirect_call_assembler(old_token, entry_loop_token) + self.cpu.redirect_call_assembler(old_token, entry_loop_token) + # entry_loop_token is also kept alive by any loop that used + # to point to old_token. Actually freeing old_token early + # is a pointless optimization (it is tiny). + old_token.record_jump_to(entry_loop_token) # ---------- @@ -239,7 +269,8 @@ if hasattr(self, 'maybe_compile_and_run'): return self.maybe_compile_and_run - metainterp_sd = self.warmrunnerdesc.metainterp_sd + warmrunnerdesc = self.warmrunnerdesc + metainterp_sd = warmrunnerdesc.metainterp_sd jitdriver_sd = self.jitdriver_sd vinfo = jitdriver_sd.virtualizable_info index_of_virtualizable = jitdriver_sd.index_of_virtualizable @@ -297,23 +328,27 @@ assert cell.counter == -1 if not confirm_enter_jit(*args): return + loop_token = cell.get_entry_loop_token() + if loop_token is None: # it was a weakref that has been freed + cell.counter = 0 + return # machine code was already compiled for these greenargs # get the assembler and fill in the boxes set_future_values(*args[num_green_args:]) - loop_token = cell.entry_loop_token # ---------- execute assembler ---------- while True: # until interrupted by an exception metainterp_sd.profiler.start_running() debug_start("jit-running") - fail_descr = metainterp_sd.cpu.execute_token(loop_token) + fail_descr = warmrunnerdesc.execute_token(loop_token) debug_stop("jit-running") metainterp_sd.profiler.end_running() + loop_token = None # for test_memmgr if vinfo is not None: vinfo.reset_vable_token(virtualizable) loop_token = fail_descr.handle_fail(metainterp_sd, jitdriver_sd) - + maybe_compile_and_run._dont_inline_ = True self.maybe_compile_and_run = maybe_compile_and_run return maybe_compile_and_run @@ -459,7 +494,7 @@ warmrunnerdesc = self.warmrunnerdesc jitdriver_sd = self.jitdriver_sd - cpu = warmrunnerdesc.cpu + cpu = self.cpu vinfo = jitdriver_sd.virtualizable_info red_args_types = unrolling_iterable(jitdriver_sd._red_args_types) # @@ -508,10 +543,11 @@ if hasattr(self, 'get_location_str'): return # + warmrunnerdesc = self.warmrunnerdesc unwrap_greenkey = self.make_unwrap_greenkey() jit_getter = self.make_jitcell_getter() jd = self.jitdriver_sd - cpu = self.warmrunnerdesc.cpu + cpu = self.cpu def can_inline_greenargs(*greenargs): if can_never_inline(*greenargs): @@ -529,11 +565,16 @@ def get_assembler_token(greenkey, redboxes): # 'redboxes' is only used to know the types of red arguments cell = self.jit_cell_at_key(greenkey) - if cell.entry_loop_token is None: + entry_loop_token = cell.get_entry_loop_token() + if entry_loop_token is None: from pypy.jit.metainterp.compile import compile_tmp_callback - cell.entry_loop_token = compile_tmp_callback(cpu, jd, greenkey, - redboxes) - return cell.entry_loop_token + if cell.counter == -1: # used to be a valid entry bridge, + cell.counter = 0 # but was freed in the meantime. + memmgr = warmrunnerdesc.memory_manager + entry_loop_token = compile_tmp_callback(cpu, jd, greenkey, + redboxes, memmgr) + cell.set_entry_loop_token(entry_loop_token) + return entry_loop_token self.get_assembler_token = get_assembler_token # Modified: pypy/branch/out-of-line-guards/pypy/jit/tl/spli/interpreter.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/tl/spli/interpreter.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/tl/spli/interpreter.py Thu Dec 9 16:33:59 2010 @@ -105,26 +105,32 @@ self.stack_depth += 1 def pop(self): - self.stack_depth -= 1 - val = self.value_stack[self.stack_depth] - self.value_stack[self.stack_depth] = None + sd = self.stack_depth - 1 + assert sd >= 0 + self.stack_depth = sd + val = self.value_stack[sd] + self.value_stack[sd] = None return val def pop_many(self, n): return [self.pop() for i in range(n)] def peek(self): - return self.value_stack[self.stack_depth - 1] + sd = self.stack_depth - 1 + assert sd >= 0 + return self.value_stack[sd] def POP_TOP(self, _, next_instr, code): self.pop() return next_instr def LOAD_FAST(self, name_index, next_instr, code): + assert name_index >= 0 self.push(self.locals[name_index]) return next_instr def STORE_FAST(self, name_index, next_instr, code): + assert name_index >= 0 self.locals[name_index] = self.pop() return next_instr Modified: pypy/branch/out-of-line-guards/pypy/jit/tl/tl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/tl/tl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/tl/tl.py Thu Dec 9 16:33:59 2010 @@ -16,32 +16,40 @@ def __init__(self, size): self = hint(self, access_directly=True, fresh_virtualizable=True) self.stack = [0] * size - self.stackpos = 0 + self.stackpos = 0 # always store a known-nonneg integer here def append(self, elem): self.stack[self.stackpos] = elem self.stackpos += 1 def pop(self): - self.stackpos -= 1 - if self.stackpos < 0: + stackpos = self.stackpos - 1 + if stackpos < 0: raise IndexError - return self.stack[self.stackpos] + self.stackpos = stackpos # always store a known-nonneg integer here + return self.stack[stackpos] def pick(self, i): - self.append(self.stack[self.stackpos - i - 1]) + n = self.stackpos - i - 1 + assert n >= 0 + self.append(self.stack[n]) def put(self, i): elem = self.pop() - self.stack[self.stackpos - i - 1] = elem + n = self.stackpos - i - 1 + assert n >= 0 + self.stack[n] = elem def roll(self, r): if r < -1: i = self.stackpos + r if i < 0: raise IndexError - elem = self.stack[self.stackpos - 1] + n = self.stackpos - 1 + assert n >= 0 + elem = self.stack[n] for j in range(self.stackpos - 2, i - 1, -1): + assert j >= 0 self.stack[j + 1] = self.stack[j] self.stack[i] = elem elif r > 1: @@ -51,7 +59,9 @@ elem = self.stack[i] for j in range(i, self.stackpos - 1): self.stack[j] = self.stack[j + 1] - self.stack[self.stackpos - 1] = elem + n = self.stackpos - 1 + assert n >= 0 + self.stack[n] = elem def make_interp(supports_call): Modified: pypy/branch/out-of-line-guards/pypy/jit/tool/jitoutput.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/tool/jitoutput.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/tool/jitoutput.py Thu Dec 9 16:33:59 2010 @@ -27,6 +27,10 @@ (('nvirtuals',), '^nvirtuals:\s+(\d+)$'), (('nvholes',), '^nvholes:\s+(\d+)$'), (('nvreused',), '^nvreused:\s+(\d+)$'), + (('total_compiled_loops',), '^Total # of loops:\s+(\d+)$'), + (('total_compiled_bridges',), '^Total # of bridges:\s+(\d+)$'), + (('total_freed_loops',), '^Freed # of loops:\s+(\d+)$'), + (('total_freed_bridges',), '^Freed # of bridges:\s+(\d+)$'), ] class Ops(object): Modified: pypy/branch/out-of-line-guards/pypy/jit/tool/test/test_jitoutput.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/tool/test/test_jitoutput.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/tool/test/test_jitoutput.py Thu Dec 9 16:33:59 2010 @@ -1,10 +1,11 @@ import py from pypy.jit.metainterp.warmspot import ll_meta_interp -from pypy.rlib.jit import JitDriver, DEBUG_PROFILE +from pypy.rlib.jit import JitDriver from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.jitprof import Profiler, JITPROF_LINES from pypy.jit.tool.jitoutput import parse_prof +from pypy.tool.logparser import parse_log, extract_category def test_really_run(): """ This test checks whether output of jitprof did not change. @@ -21,13 +22,15 @@ cap = py.io.StdCaptureFD() try: ll_meta_interp(f, [10], CPUClass=runner.LLtypeCPU, type_system='lltype', - ProfilerClass=Profiler, debug_level=DEBUG_PROFILE) + ProfilerClass=Profiler) finally: out, err = cap.reset() - err = "\n".join(err.splitlines()[-JITPROF_LINES:]) - print err - assert err.count("\n") == JITPROF_LINES - 1 - info = parse_prof(err) + + log = parse_log(err.splitlines(True)) + err_sections = list(extract_category(log, 'jit-summary')) + [err1] = err_sections # there should be exactly one jit-summary + assert err1.count("\n") == JITPROF_LINES + info = parse_prof(err1) # assert did not crash # asserts below are a bit delicate, possibly they might be deleted assert info.tracing_no == 1 @@ -60,6 +63,10 @@ nvirtuals: 13 nvholes: 14 nvreused: 15 +Total # of loops: 100 +Total # of bridges: 300 +Freed # of loops: 99 +Freed # of bridges: 299 ''' def test_parse(): Modified: pypy/branch/out-of-line-guards/pypy/module/__pypy__/__init__.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/__pypy__/__init__.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/__pypy__/__init__.py Thu Dec 9 16:33:59 2010 @@ -11,6 +11,11 @@ 'internal_repr' : 'interp_magic.internal_repr', 'bytebuffer' : 'bytebuffer.bytebuffer', 'identity_dict' : 'interp_identitydict.W_IdentityDict', + 'debug_start' : 'interp_debug.debug_start', + 'debug_print' : 'interp_debug.debug_print', + 'debug_stop' : 'interp_debug.debug_stop', + 'debug_print_once' : 'interp_debug.debug_print_once', + 'builtinify' : 'interp_magic.builtinify', } def setup_after_space_initialization(self): Modified: pypy/branch/out-of-line-guards/pypy/module/__pypy__/interp_magic.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/__pypy__/interp_magic.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/__pypy__/interp_magic.py Thu Dec 9 16:33:59 2010 @@ -51,3 +51,9 @@ return space.newtuple([space.newint(cache.hits.get(name, 0)), space.newint(cache.misses.get(name, 0))]) mapdict_cache_counter.unwrap_spec = [ObjSpace, str] + +def builtinify(space, w_func): + from pypy.interpreter.function import Function, BuiltinFunction + func = space.interp_w(Function, w_func) + bltn = BuiltinFunction(func) + return space.wrap(bltn) Modified: pypy/branch/out-of-line-guards/pypy/module/__pypy__/test/test_special.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/__pypy__/test/test_special.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/__pypy__/test/test_special.py Thu Dec 9 16:33:59 2010 @@ -21,3 +21,18 @@ def test_cpumodel(self): import __pypy__ assert hasattr(__pypy__, 'cpumodel') + + def test_builtinify(self): + import __pypy__ + class A(object): + a = lambda *args: args + b = __pypy__.builtinify(a) + my = A() + assert my.a() == (my,) + assert my.b() == () + assert A.a(my) == (my,) + assert A.b(my) == (my,) + assert A.a.im_func(my) == (my,) + assert not hasattr(A.b, 'im_func') + assert A.a is not A.__dict__['a'] + assert A.b is A.__dict__['b'] Modified: pypy/branch/out-of-line-guards/pypy/module/_lsprof/interp_lsprof.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/_lsprof/interp_lsprof.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/_lsprof/interp_lsprof.py Thu Dec 9 16:33:59 2010 @@ -163,8 +163,11 @@ if isinstance(w_arg, Method): w_function = w_arg.w_function class_name = w_arg.w_class.getname(space, '?') - assert isinstance(w_function, Function) - return "{method '%s' of '%s' objects}" % (w_function.name, class_name) + if isinstance(w_function, Function): + name = w_function.name + else: + name = '?' + return "{method '%s' of '%s' objects}" % (name, class_name) elif isinstance(w_arg, Function): if w_arg.w_module is None: module = '' @@ -176,7 +179,8 @@ module += '.' return '{%s%s}' % (module, w_arg.name) else: - return '{!!!unknown!!!}' + class_name = space.type(w_arg).getname(space, '?') + return "{'%s' object}" % (class_name,) def lsprof_call(space, w_self, frame, event, w_arg): assert isinstance(w_self, W_Profiler) Modified: pypy/branch/out-of-line-guards/pypy/module/_lsprof/test/test_cprofile.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/_lsprof/test/test_cprofile.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/_lsprof/test/test_cprofile.py Thu Dec 9 16:33:59 2010 @@ -2,9 +2,10 @@ from pypy.conftest import gettestobjspace, option class AppTestCProfile(object): + keywords = {} def setup_class(cls): - space = gettestobjspace(usemodules=('_lsprof',)) + space = gettestobjspace(usemodules=('_lsprof',), **cls.keywords) cls.w_expected_output = space.wrap(expected_output) cls.space = space cls.w_file = space.wrap(__file__) @@ -148,6 +149,12 @@ finally: sys.path.pop(0) + +class AppTestWithDifferentBytecodes(AppTestCProfile): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True, + 'objspace.opcodes.CALL_METHOD': True} + + expected_output = {} expected_output['print_stats'] = """\ 126 function calls (106 primitive calls) in 1.000 CPU seconds @@ -165,11 +172,11 @@ 2 0.000 0.000 0.140 0.070 profilee.py:84(helper2_indirect) 8 0.312 0.039 0.400 0.050 profilee.py:88(helper2) 8 0.064 0.008 0.080 0.010 profilee.py:98(subhelper) - 4 0.000 0.000 0.000 0.000 {.*append.*} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} 1 0.000 0.000 0.000 0.000 {.*disable.*} - 12 0.000 0.000 0.012 0.001 {hasattr.*} - 8 0.000 0.000 0.000 0.000 {range.*} - 4 0.000 0.000 0.000 0.000 {sys.exc_info.*} + 12 0.000 0.000 0.012 0.001 {hasattr} + 8 0.000 0.000 0.000 0.000 {range} + 4 0.000 0.000 0.000 0.000 {sys.exc_info} """ Modified: pypy/branch/out-of-line-guards/pypy/module/_minimal_curses/__init__.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/_minimal_curses/__init__.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/_minimal_curses/__init__.py Thu Dec 9 16:33:59 2010 @@ -4,7 +4,7 @@ try: import _minimal_curses as _curses # when running on top of pypy-c except ImportError: - import py; py.test.skip("no _curses module") # no _curses at all + raise ImportError("no _curses or _minimal_curses module") # no _curses at all from pypy.interpreter.mixedmodule import MixedModule from pypy.module._minimal_curses import fficurses Modified: pypy/branch/out-of-line-guards/pypy/module/_pickle_support/maker.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/_pickle_support/maker.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/_pickle_support/maker.py Thu Dec 9 16:33:59 2010 @@ -67,11 +67,12 @@ return space.wrap(tb) traceback_new.unwrap_spec = [ObjSpace] -def generator_new(space, frame, running): +def generator_new(space, w_frame, running): + frame = space.interp_w(PyFrame, w_frame, can_be_None=True) new_generator = GeneratorIterator(frame) new_generator.running = running return space.wrap(new_generator) -generator_new.unwrap_spec = [ObjSpace, PyFrame, int] +generator_new.unwrap_spec = [ObjSpace, W_Root, int] def xrangeiter_new(space, current, remaining, step): from pypy.module.__builtin__.functional import W_XRangeIterator Modified: pypy/branch/out-of-line-guards/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/_stackless/interp_coroutine.py Thu Dec 9 16:33:59 2010 @@ -304,16 +304,19 @@ def w_descr__framestack(space, self): assert isinstance(self, AppCoroutine) - index = self.subctx.framestackdepth - if not index: - return space.newtuple([]) - items = [None] * index + counter = 0 f = self.subctx.topframe - while index > 0: - index -= 1 - items[index] = space.wrap(f) + while f is not None: + counter += 1 f = f.f_backref() - assert f is None + items = [None] * counter + f = self.subctx.topframe + while f is not None: + counter -= 1 + assert counter >= 0 + items[counter] = space.wrap(f) + f = f.f_backref() + assert counter == 0 return space.newtuple(items) def makeStaticMethod(module, classname, funcname): Modified: pypy/branch/out-of-line-guards/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/array/test/test_array.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/array/test/test_array.py Thu Dec 9 16:33:59 2010 @@ -64,6 +64,7 @@ raises(TypeError, self.array, tc, None) def test_value_range(self): + import sys values = (-129, 128, -128, 127, 0, 255, -1, 256, -32768, 32767, -32769, 32768, 65535, 65536, -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, @@ -88,7 +89,12 @@ a.append(v) for i, v in enumerate(ok * 2): assert a[i] == v - assert type(a[i]) is pt + assert type(a[i]) is pt or ( + # A special case: we return ints in Array('I') on 64-bits, + # whereas CPython returns longs. The difference is + # probably acceptable. + tc == 'I' and + sys.maxint > 2147483647 and type(a[i]) is int) for v in ok: a[1] = v assert a[0] == ok[0] Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/__init__.py Thu Dec 9 16:33:59 2010 @@ -69,6 +69,7 @@ import pypy.module.cpyext.weakrefobject import pypy.module.cpyext.funcobject import pypy.module.cpyext.classobject +import pypy.module.cpyext.pypyintf # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/api.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/api.py Thu Dec 9 16:33:59 2010 @@ -321,6 +321,8 @@ 'PyCObject_Type', 'init_pycobject', 'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer', + + 'PyStructSequence_InitType', 'PyStructSequence_New', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur @@ -845,6 +847,7 @@ source_dir / "bufferobject.c", source_dir / "object.c", source_dir / "cobject.c", + source_dir / "structseq.c", ], separate_module_sources = [code, struct_source], export_symbols=export_symbols_eci, Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/cdatetime.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/cdatetime.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/cdatetime.py Thu Dec 9 16:33:59 2010 @@ -4,8 +4,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, cpython_struct, PyObjectFields) from pypy.module.cpyext.import_ import PyImport_Import -from pypy.module.cpyext.typeobject import PyTypeObjectPtr, render_immortal -from pypy.module.cpyext.state import State +from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_renamer @@ -25,48 +24,26 @@ datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw', track_allocation=False) - if not we_are_translated(): - datetimeAPI_dealloc(space) - space.fromcache(State).datetimeAPI = datetimeAPI - w_datetime = PyImport_Import(space, space.wrap("datetime")) w_type = space.getattr(w_datetime, space.wrap("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DateType, w_type) w_type = space.getattr(w_datetime, space.wrap("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DateTimeType, w_type) w_type = space.getattr(w_datetime, space.wrap("time")) datetimeAPI.c_TimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_TimeType, w_type) w_type = space.getattr(w_datetime, space.wrap("timedelta")) datetimeAPI.c_DeltaType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DeltaType, w_type) return datetimeAPI -def datetimeAPI_dealloc(space): - "Used in tests only, to please the refcount checker" - if we_are_translated(): - return - datetimeAPI = space.fromcache(State).datetimeAPI - if datetimeAPI is None: - return - space.fromcache(State).datetimeAPI = None - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DateType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DateTimeType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_TimeType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DeltaType)) - lltype.free(datetimeAPI, flavor='raw') - PyDateTime_Date = PyObject PyDateTime_Time = PyObject PyDateTime_DateTime = PyObject Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/include/Python.h Thu Dec 9 16:33:59 2010 @@ -8,6 +8,8 @@ # include # include # include +# include +# include # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_FUNC(RTYPE) RTYPE # define PyAPI_DATA(RTYPE) extern RTYPE @@ -103,6 +105,7 @@ #include "sliceobject.h" #include "datetime.h" #include "pystate.h" +#include "fileobject.h" // XXX This shouldn't be included here #include "structmember.h" @@ -120,4 +123,8 @@ #define PyDoc_STR(str) "" #endif +/* PyPy does not implement --with-fpectl */ +#define PyFPE_START_PROTECT(err_string, leave_stmt) +#define PyFPE_END_PROTECT(v) + #endif Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/intobject.py Thu Dec 9 16:33:59 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, build_type_checkers, Py_ssize_t) @@ -19,6 +20,9 @@ already one, and then return its value. If there is an error, -1 is returned, and the caller should check PyErr_Occurred() to find out whether there was an error, or whether the value just happened to be -1.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Unsigned, error=-1) @@ -26,6 +30,9 @@ """Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.uint_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Signed, error=CANNOT_FAIL) @@ -39,6 +46,9 @@ PyLongObject, if it is not already one, and then return its value as Py_ssize_t. """ + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(w_obj) # XXX this is wrong on win64 @cpython_api([Py_ssize_t], PyObject) @@ -48,4 +58,3 @@ returned. """ return space.wrap(ival) # XXX this is wrong on win64 - Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/object.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/object.py Thu Dec 9 16:33:59 2010 @@ -2,12 +2,13 @@ from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, - Py_GE, CONST_STRING, FILEP, fwrite) + Py_GE, CONST_STRING, FILEP, fwrite, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference, get_typedescr, RefcountState) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall +from pypy.module._file.interp_file import W_File from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject from pypy.interpreter.error import OperationError @@ -428,6 +429,34 @@ rffi.free_nonmovingbuffer(data, buf) return 0 +PyFile_Check, PyFile_CheckExact = build_type_checkers("File", W_File) + + at cpython_api([PyObject, rffi.INT_real], PyObject) +def PyFile_GetLine(space, w_obj, n): + """ + Equivalent to p.readline([n]), this function reads one line from the + object p. p may be a file object or any object with a readline() + method. If n is 0, exactly one line is read, regardless of the length of + the line. If n is greater than 0, no more than n bytes will be read + from the file; a partial line can be returned. In both cases, an empty string + is returned if the end of the file is reached immediately. If n is less than + 0, however, one line is read regardless of length, but EOFError is + raised if the end of the file is reached immediately.""" + try: + w_readline = space.getattr(w_obj, space.wrap('readline')) + except OperationError: + raise OperationError( + space.w_TypeError, space.wrap( + "argument must be a file, or have a readline() method.")) + + n = rffi.cast(lltype.Signed, n) + if space.is_true(space.gt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline, space.wrap(n)) + elif space.is_true(space.lt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline) + else: + # XXX Raise EOFError as specified + return space.call_function(w_readline) @cpython_api([CONST_STRING, CONST_STRING], PyObject) def PyFile_FromString(space, filename, mode): """ @@ -437,4 +466,3 @@ w_filename = space.wrap(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) - Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/pyerrors.py Thu Dec 9 16:33:59 2010 @@ -5,7 +5,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, Py_DecRef, borrow_from) + PyObject, PyObjectP, make_ref, from_ref, Py_DecRef, borrow_from) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from pypy.rlib.rposix import get_errno @@ -80,6 +80,21 @@ Py_DecRef(space, w_value) Py_DecRef(space, w_traceback) + at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) +def PyErr_NormalizeException(space, exc_p, val_p, tb_p): + """Under certain circumstances, the values returned by PyErr_Fetch() below + can be "unnormalized", meaning that *exc is a class object but *val is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance.""" + operr = OperationError(from_ref(space, exc_p[0]), + from_ref(space, val_p[0])) + operr.normalize_exception(space) + Py_DecRef(space, exc_p[0]) + Py_DecRef(space, val_p[0]) + exc_p[0] = make_ref(space, operr.w_type) + val_p[0] = make_ref(space, operr.get_w_value(space)) + @cpython_api([], lltype.Void) def PyErr_BadArgument(space): """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where @@ -114,10 +129,29 @@ function around a system call can write return PyErr_SetFromErrno(type); when the system call returns an error. Return value: always NULL.""" + PyErr_SetFromErrnoWithFilename(space, w_type, + lltype.nullptr(rffi.CCHARP.TO)) + + at cpython_api([PyObject, rffi.CCHARP], PyObject) +def PyErr_SetFromErrnoWithFilename(space, w_type, llfilename): + """Similar to PyErr_SetFromErrno(), with the additional behavior that if + filename is not NULL, it is passed to the constructor of type as a third + parameter. In the case of exceptions such as IOError and OSError, + this is used to define the filename attribute of the exception instance. + Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() msg = os.strerror(errno) - w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + if llfilename: + w_filename = rffi.charp2str(llfilename) + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg), + space.wrap(w_filename)) + else: + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg)) raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/pythonrun.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/pythonrun.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/pythonrun.py Thu Dec 9 16:33:59 2010 @@ -1,6 +1,16 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.state import State @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def Py_IsInitialized(space): return 1 + + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) +def Py_GetProgramName(space): + """ + Return the program name set with Py_SetProgramName(), or the default. + The returned string points into static storage; the caller should not modify its + value.""" + return space.fromcache(State).get_programname() + Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/sequence.py Thu Dec 9 16:33:59 2010 @@ -116,6 +116,14 @@ This is the equivalent of the Python expression o1 + o2.""" return space.add(w_o1, w_o2) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PySequence_Contains(space, w_obj, w_value): + """Determine if o contains value. If an item in o is equal to value, + return 1, otherwise return 0. On error, return -1. This is + equivalent to the Python expression value in o.""" + w_res = space.contains(w_obj, w_value) + return space.int_w(w_res) + @cpython_api([PyObject], PyObject) def PySeqIter_New(space, w_seq): """Return an iterator that works with a general sequence object, seq. The Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/slotdefs.py Thu Dec 9 16:33:59 2010 @@ -6,7 +6,7 @@ unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, setattrofunc, lenfunc, ssizeargfunc, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, hashfunc, - descrgetfunc, descrsetfunc) + descrgetfunc, descrsetfunc, objobjproc) from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -156,6 +156,15 @@ if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) +def wrap_objobjproc(space, w_self, w_args, func): + func_target = rffi.cast(objobjproc, func) + check_num_args(space, w_args, 1) + w_value, = space.fixedview(w_args) + res = generic_cpy_call(space, func_target, w_self, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.wrap(res) + def wrap_ssizessizeargfunc(space, w_self, w_args, func): func_target = rffi.cast(ssizessizeargfunc, func) check_num_args(space, w_args, 2) Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/src/getargs.c Thu Dec 9 16:33:59 2010 @@ -453,6 +453,7 @@ strncpy(msgbuf, "is not retrievable", bufsize); return msgbuf; } + PyPy_Borrow(arg, item); msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); /* PySequence_GetItem calls tp->sq_item, which INCREFs */ Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/state.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/state.py Thu Dec 9 16:33:59 2010 @@ -5,11 +5,10 @@ class State: - datetimeAPI = None # used in tests - def __init__(self, space): self.space = space self.reset() + self.programname = lltype.nullptr(rffi.CCHARP.TO) def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef @@ -43,3 +42,16 @@ if always: raise OperationError(self.space.w_SystemError, self.space.wrap( "Function returned an error result without setting an exception")) + + def get_programname(self): + if not self.programname: + space = self.space + argv = space.sys.get('argv') + if space.int_w(space.len(argv)): + argv0 = space.getitem(argv, space.wrap(0)) + progname = space.str_w(argv0) + else: + progname = "pypy" + self.programname = rffi.str2charp(progname) + lltype.render_immortal(self.programname) + return self.programname Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/stubs.py Thu Dec 9 16:33:59 2010 @@ -668,24 +668,6 @@ """ raise NotImplementedError - at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) -def PyErr_NormalizeException(space, exc, val, tb): - """Under certain circumstances, the values returned by PyErr_Fetch() below - can be "unnormalized", meaning that *exc is a class object but *val is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], PyObject) -def PyErr_SetFromErrnoWithFilename(space, type, filename): - """Similar to PyErr_SetFromErrno(), with the additional behavior that if - filename is not NULL, it is passed to the constructor of type as a third - parameter. In the case of exceptions such as IOError and OSError, - this is used to define the filename attribute of the exception instance. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([rffi.INT_real], PyObject) def PyErr_SetFromWindowsErr(space, ierr): """This is a convenience function to raise WindowsError. If called with @@ -809,21 +791,6 @@ successful invocation of Py_EnterRecursiveCall().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_Check(space, p): - """Return true if its argument is a PyFileObject or a subtype of - PyFileObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_CheckExact(space, p): - """Return true if its argument is a PyFileObject, but not a subtype of - PyFileObject. - """ - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file @@ -857,22 +824,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], PyObject) -def PyFile_GetLine(space, p, n): - """ - - - - Equivalent to p.readline([n]), this function reads one line from the - object p. p may be a file object or any object with a readline() - method. If n is 0, exactly one line is read, regardless of the length of - the line. If n is greater than 0, no more than n bytes will be read - from the file; a partial line can be returned. In both cases, an empty string - is returned if the end of the file is reached immediately. If n is less than - 0, however, one line is read regardless of length, but EOFError is - raised if the end of the file is reached immediately.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyFile_Name(space, p): """Return the name of the file specified by p as a string object.""" @@ -1431,17 +1382,6 @@ raise NotImplementedError @cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) -def Py_GetProgramName(space, ): - """ - - - - Return the program name set with Py_SetProgramName(), or the default. - The returned string points into static storage; the caller should not modify its - value.""" - raise NotImplementedError - - at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPrefix(space, ): """Return the prefix for installed platform-independent files. This is derived through a number of complicated rules from the program name set with @@ -2289,13 +2229,6 @@ in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PySequence_Contains(space, o, value): - """Determine if o contains value. If an item in o is equal to value, - return 1, otherwise return 0. On error, return -1. This is - equivalent to the Python expression value in o.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PySequence_Index(space, o, value): """Return the first index i for which o[i] == value. On error, return Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_arraymodule.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_arraymodule.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_arraymodule.py Thu Dec 9 16:33:59 2010 @@ -14,6 +14,7 @@ arr.append(4) assert arr.tolist() == [1, 2, 3, 4] assert len(arr) == 4 + self.cleanup_references() def test_iter(self): module = self.import_module(name='array') @@ -22,6 +23,7 @@ for i in arr: sum += i assert sum == 6 + self.cleanup_references() def test_index(self): module = self.import_module(name='array') @@ -32,6 +34,7 @@ assert arr.tolist() == [1,2,4] arr[2] = 99 assert arr.tolist() == [1,2,99] + self.cleanup_references() def test_slice_get(self): module = self.import_module(name='array') @@ -40,3 +43,4 @@ assert arr[1:].tolist() == [2,3,4] assert arr[:2].tolist() == [1,2] assert arr[1:3].tolist() == [2,3] + self.cleanup_references() Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_cpyext.py Thu Dec 9 16:33:59 2010 @@ -1,10 +1,12 @@ import sys +import weakref import os.path import py from pypy.conftest import gettestobjspace from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import interp2app from pypy.rpython.lltypesystem import rffi, lltype, ll2ctypes from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator import platform @@ -79,6 +81,23 @@ self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) class LeakCheckingTest(object): + @staticmethod + def cleanup_references(space): + state = space.fromcache(RefcountState) + + import gc; gc.collect() + # Clear all lifelines, objects won't resurrect + for w_obj, obj in state.lifeline_dict._dict.items(): + if w_obj not in state.py_objects_w2r: + state.lifeline_dict.set(w_obj, None) + del obj + import gc; gc.collect() + + for w_obj in state.non_heaptypes_w: + Py_DecRef(space, w_obj) + state.non_heaptypes_w[:] = [] + state.reset_borrowed_references() + def check_and_print_leaks(self): # check for sane refcnts import gc @@ -89,13 +108,6 @@ lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) - # Clear all lifelines, objects won't resurrect - for w_obj, obj in state.lifeline_dict._dict.items(): - if w_obj not in state.py_objects_w2r: - state.lifeline_dict.set(w_obj, None) - del obj - gc.collect() - for w_obj, obj in state.py_objects_w2r.iteritems(): base_refcnt = self.frozen_refcounts.get(w_obj) delta = obj.c_ob_refcnt @@ -105,7 +117,12 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = state.lifeline_dict.get(w_obj) + try: + weakref.ref(w_obj) + except TypeError: + lifeline = None + else: + lifeline = state.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: @@ -137,6 +154,8 @@ state = cls.space.fromcache(RefcountState) state.non_heaptypes_w[:] = [] + cls.w_cleanup_references = cls.space.wrap(interp2app(cls.cleanup_references)) + def compile_module(self, name, **kwds): """ Build an extension module linked against the cpyext api library. @@ -256,13 +275,7 @@ def teardown_method(self, func): for name in self.imported_module_names: self.unimport_module(name) - state = self.space.fromcache(RefcountState) - for w_obj in state.non_heaptypes_w: - Py_DecRef(self.space, w_obj) - state.non_heaptypes_w[:] = [] - state.reset_borrowed_references() - from pypy.module.cpyext import cdatetime - cdatetime.datetimeAPI_dealloc(self.space) + self.cleanup_references(self.space) if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." @@ -669,3 +682,19 @@ assert mod.get_names() == ('cell', 'module', 'property', 'staticmethod', 'builtin_function_or_method') + + def test_get_programname(self): + mod = self.import_extension('foo', [ + ('get_programname', 'METH_NOARGS', + ''' + char* name1 = Py_GetProgramName(); + char* name2 = Py_GetProgramName(); + if (name1 != name2) + Py_RETURN_FALSE; + return PyString_FromString(name1); + ''' + ), + ]) + p = mod.get_programname() + print p + assert 'py' in p Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_datetime.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_datetime.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_datetime.py Thu Dec 9 16:33:59 2010 @@ -92,9 +92,20 @@ PyDateTimeAPI->TimeType, PyDateTimeAPI->DeltaType); """), + ("clear_types", "METH_NOARGS", + """ + Py_DECREF(PyDateTimeAPI->DateType); + Py_DECREF(PyDateTimeAPI->DateTimeType); + Py_DECREF(PyDateTimeAPI->TimeType); + Py_DECREF(PyDateTimeAPI->DeltaType); + Py_RETURN_NONE; + """ + ) ]) import datetime assert module.get_types() == (datetime.date, datetime.datetime, datetime.time, datetime.timedelta) + module.clear_types() + self.cleanup_references() Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_intobject.py Thu Dec 9 16:33:59 2010 @@ -19,6 +19,10 @@ assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + assert api.PyInt_AsLong(None) == -1 + assert api.PyErr_Occurred() is space.w_TypeError + api.PyErr_Clear() + assert api.PyInt_AsUnsignedLong(space.wrap(sys.maxint)) == sys.maxint assert api.PyInt_AsUnsignedLong(space.wrap(-5)) == sys.maxint * 2 + 1 assert api.PyErr_Occurred() is space.w_ValueError Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_object.py Thu Dec 9 16:33:59 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ - Py_GE, Py_GT + Py_GE, Py_GT, fopen, fclose, fwrite from pypy.tool.udir import udir class TestObject(BaseApiTest): @@ -188,10 +188,45 @@ rffi.free_charp(filename) rffi.free_charp(mode) + assert api.PyFile_Check(w_file) + assert api.PyFile_CheckExact(w_file) + assert not api.PyFile_Check(space.wrap("text")) + space.call_method(w_file, "write", space.wrap("text")) space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" + def test_file_getline(self, space, api): + filename = rffi.str2charp(str(udir / "_test_file")) + + mode = rffi.str2charp("w") + w_file = api.PyFile_FromString(filename, mode) + space.call_method(w_file, "write", + space.wrap("line1\nline2\nline3\nline4")) + space.call_method(w_file, "close") + + rffi.free_charp(mode) + mode = rffi.str2charp("r") + w_file = api.PyFile_FromString(filename, mode) + rffi.free_charp(filename) + rffi.free_charp(mode) + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "line1\n" + + w_line = api.PyFile_GetLine(w_file, 4) + assert space.str_w(w_line) == "line" + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "2\n" + + # XXX We ought to raise an EOFError here, but don't + w_line = api.PyFile_GetLine(w_file, -1) + # assert api.PyErr_Occurred() is space.w_EOFError + assert space.str_w(w_line) == "line3\n" + + space.call_method(w_file, "close") + class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_pyerrors.py Thu Dec 9 16:33:59 2010 @@ -129,6 +129,41 @@ ]) assert module.check_error() + + def test_normalize(self): + module = self.import_extension('foo', [ + ("check_error", "METH_NOARGS", + ''' + PyObject *type, *val, *tb; + PyErr_SetString(PyExc_TypeError, "message"); + + PyErr_Fetch(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (!PyString_Check(val)) + Py_RETURN_FALSE; + /* Normalize */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + /* Normalize again */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + PyErr_Restore(type, val, tb); + PyErr_Clear(); + Py_RETURN_TRUE; + ''' + ), + ]) + assert module.check_error() + def test_SetFromErrno(self): import sys if sys.platform != 'win32': @@ -149,3 +184,26 @@ except OSError, e: assert e.errno == errno.EBADF assert e.strerror == os.strerror(errno.EBADF) + assert e.filename == None + + def test_SetFromErrnoWithFilename(self): + import sys + if sys.platform != 'win32': + skip("callbacks through ll2ctypes modify errno") + import errno, os + + module = self.import_extension('foo', [ + ("set_from_errno", "METH_NOARGS", + ''' + errno = EBADF; + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "blyf"); + return NULL; + '''), + ], + prologue="#include ") + try: + module.set_from_errno() + except OSError, e: + assert e.filename == "blyf" + assert e.errno == errno.EBADF + assert e.strerror == os.strerror(errno.EBADF) Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_sequence.py Thu Dec 9 16:33:59 2010 @@ -70,3 +70,11 @@ assert space.unwrap(space.next(w_iter)) == 2 exc = raises(OperationError, space.next, w_iter) assert exc.value.match(space, space.w_StopIteration) + + def test_contains(self, space, api): + w_t = space.wrap((1, 'ha')) + assert api.PySequence_Contains(w_t, space.wrap(u'ha')) + assert not api.PySequence_Contains(w_t, space.wrap(2)) + assert api.PySequence_Contains(space.w_None, space.wrap(2)) == -1 + assert api.PyErr_Occurred() + api.PyErr_Clear() Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/test/test_typeobject.py Thu Dec 9 16:33:59 2010 @@ -20,6 +20,7 @@ assert type(obj) is module.fooType print "type of obj has type", type(type(obj)) print "type of type of obj has type", type(type(type(obj))) + self.cleanup_references() def test_typeobject_method_descriptor(self): module = self.import_module(name='foo') @@ -38,6 +39,7 @@ print obj.foo assert obj.foo == 42 assert obj.int_member == obj.foo + self.cleanup_references() def test_typeobject_data_member(self): module = self.import_module(name='foo') @@ -54,6 +56,7 @@ raises(SystemError, "obj.broken_member = 42") assert module.fooType.broken_member.__doc__ is None assert module.fooType.object_member.__doc__ == "A Python object." + self.cleanup_references() def test_typeobject_object_member(self): module = self.import_module(name='foo') @@ -74,6 +77,7 @@ obj.set_foo = 32 assert obj.foo == 32 + self.cleanup_references() def test_typeobject_string_member(self): module = self.import_module(name='foo') @@ -91,6 +95,7 @@ assert obj.char_member == "a" raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") + self.cleanup_references() def test_staticmethod(self): module = self.import_module(name="foo") @@ -98,6 +103,7 @@ assert obj.foo == 42 obj2 = obj.create() assert obj2.foo == 42 + self.cleanup_references() def test_new(self): module = self.import_module(name='foo') @@ -118,7 +124,8 @@ return self assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) - + self.cleanup_references() + def test_init(self): module = self.import_module(name="foo") newobj = module.FuuType() @@ -137,6 +144,7 @@ newobj = Fuu2() assert newobj.get_val() == 42 assert newobj.foobar == 32 + self.cleanup_references() def test_metatype(self): module = self.import_module(name='foo') @@ -145,6 +153,7 @@ assert isinstance(x, type) assert isinstance(x, module.MetaType) x() + self.cleanup_references() def test_metaclass_compatible(self): # metaclasses should not conflict here @@ -153,7 +162,9 @@ assert type(module.fooType).__mro__ == (type, object) y = module.MetaType('other', (module.fooType,), {}) assert isinstance(y, module.MetaType) - y() + x = y() + del x, y + self.cleanup_references() def test_sre(self): module = self.import_module(name='_sre') @@ -172,17 +183,21 @@ assert "groupdict" in dir(m) re._cache.clear() re._cache_repl.clear() + del prog, m + self.cleanup_references() def test_init_error(self): module = self.import_module("foo") raises(ValueError, module.InitErrType) - + self.cleanup_references() + def test_cmps(self): module = self.import_module("comparisons") cmpr = module.CmpType() assert cmpr == 3 assert cmpr != 42 - + self.cleanup_references() + def test_hash(self): module = self.import_module("comparisons") cmpr = module.CmpType() @@ -191,6 +206,7 @@ d[cmpr] = 72 assert d[cmpr] == 72 assert d[3] == 72 + self.cleanup_references() def test_descriptor(self): module = self.import_module("foo") @@ -205,6 +221,7 @@ assert obj.y == (prop, 2) del obj.x assert obj.z == prop + self.cleanup_references() def test_tp_dict(self): foo = self.import_module("foo") @@ -226,6 +243,7 @@ ]) obj = foo.new() assert module.read_tp_dict(obj) == foo.fooType.copy + self.cleanup_references() class TestTypes(BaseApiTest): Modified: pypy/branch/out-of-line-guards/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/cpyext/typeobject.py Thu Dec 9 16:33:59 2010 @@ -24,7 +24,7 @@ from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, - PyNumberMethods) + PyNumberMethods, PySequenceMethods) from pypy.module.cpyext.slotdefs import ( slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) from pypy.interpreter.error import OperationError @@ -136,6 +136,8 @@ if not struct: if slot_names[0] == 'c_tp_as_number': STRUCT_TYPE = PyNumberMethods + elif slot_names[0] == 'c_tp_as_sequence': + STRUCT_TYPE = PySequenceMethods else: raise AssertionError( "Structure not allocated: %s" % (slot_names[0],)) @@ -392,6 +394,8 @@ lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') if obj_pto.c_tp_as_number: lltype.free(obj_pto.c_tp_as_number, flavor='raw') + if obj_pto.c_tp_as_sequence: + lltype.free(obj_pto.c_tp_as_sequence, flavor='raw') Py_DecRef(space, base_pyo) rffi.free_charp(obj_pto.c_tp_name) PyObject_dealloc(space, obj) @@ -433,12 +437,6 @@ finish_type_1(space, pto) finish_type_2(space, pto, w_type) - if space.type(w_type).is_cpytype(): - # XXX Types with a C metatype are never freed, try to see why... - render_immortal(pto, w_type) - lltype.render_immortal(pto) - lltype.render_immortal(pto.c_tp_name) - pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: @@ -544,25 +542,12 @@ w_obj.ready() finish_type_2(space, py_type, w_obj) - render_immortal(py_type, w_obj) state = space.fromcache(RefcountState) state.non_heaptypes_w.append(w_obj) return w_obj -def render_immortal(py_type, w_obj): - lltype.render_immortal(py_type.c_tp_bases) - lltype.render_immortal(py_type.c_tp_mro) - - assert isinstance(w_obj, W_TypeObject) - if w_obj.is_cpytype(): - lltype.render_immortal(py_type.c_tp_dict) - else: - lltype.render_immortal(py_type.c_tp_name) - if not w_obj.is_cpytype() and w_obj.is_heaptype(): - lltype.render_immortal(py_type) - def finish_type_1(space, pto): """ Sets up tp_bases, necessary before creating the interpreter type. @@ -599,7 +584,8 @@ if w_obj.is_cpytype(): Py_DecRef(space, pto.c_tp_dict) - pto.c_tp_dict = make_ref(space, w_obj.getdict()) + w_dict = space.newdict(from_strdict_shared=w_obj.dict_w) + pto.c_tp_dict = make_ref(space, w_dict) @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): Modified: pypy/branch/out-of-line-guards/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/exceptions/interp_exceptions.py Thu Dec 9 16:33:59 2010 @@ -452,6 +452,7 @@ self.w_text = space.w_None self.w_msg = space.wrap('') self.w_print_file_and_line = space.w_None # what's that? + self.w_lastlineno = space.w_None # this is a pypy extension W_BaseException.__init__(self, space) def descr_init(self, space, args_w): @@ -459,11 +460,16 @@ if len(args_w) > 0: self.w_msg = args_w[0] if len(args_w) == 2: - values_w = space.fixedview(args_w[1], 4) - self.w_filename = values_w[0] - self.w_lineno = values_w[1] - self.w_offset = values_w[2] - self.w_text = values_w[3] + values_w = space.fixedview(args_w[1]) + if len(values_w) > 0: self.w_filename = values_w[0] + if len(values_w) > 1: self.w_lineno = values_w[1] + if len(values_w) > 2: self.w_offset = values_w[2] + if len(values_w) > 3: self.w_text = values_w[3] + if len(values_w) > 4: + self.w_lastlineno = values_w[4] # PyPy extension + # kill the extra items from args_w to prevent undesired effects + args_w = args_w[:] + args_w[1] = space.newtuple(values_w[:4]) W_BaseException.descr_init(self, space, args_w) descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] @@ -472,18 +478,24 @@ if type(self.msg) is not str: return str(self.msg) + lineno = None buffer = self.msg have_filename = type(self.filename) is str - have_lineno = type(self.lineno) is int + if type(self.lineno) is int: + if (type(self.lastlineno) is int and + self.lastlineno > self.lineno): + lineno = 'lines %d-%d' % (self.lineno, self.lastlineno) + else: + lineno = 'line %d' % (self.lineno,) if have_filename: import os fname = os.path.basename(self.filename or "???") - if have_lineno: - buffer = "%s (%s, line %ld)" % (self.msg, fname, self.lineno) + if lineno: + buffer = "%s (%s, %s)" % (self.msg, fname, lineno) else: buffer ="%s (%s)" % (self.msg, fname) - elif have_lineno: - buffer = "%s (line %ld)" % (self.msg, self.lineno) + elif lineno: + buffer = "%s (%s)" % (self.msg, lineno) return buffer """) @@ -504,6 +516,7 @@ text = readwrite_attrproperty_w('w_text', W_SyntaxError), print_file_and_line = readwrite_attrproperty_w('w_print_file_and_line', W_SyntaxError), + lastlineno = readwrite_attrproperty_w('w_lastlineno', W_SyntaxError), ) W_FutureWarning = _new_exception('FutureWarning', W_Warning, Modified: pypy/branch/out-of-line-guards/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/fcntl/test/test_fcntl.py Thu Dec 9 16:33:59 2010 @@ -63,7 +63,7 @@ if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', 'Darwin1.2', 'darwin', 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', + 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', 'bsdos2', 'bsdos3', 'bsdos4', 'openbsd', 'openbsd2', 'openbsd3'): if struct.calcsize('l') == 8: @@ -159,7 +159,7 @@ if "linux" in sys.platform: TIOCGPGRP = 0x540f - elif "darwin" in sys.platform or "freebsd6" == sys.platform: + elif "darwin" in sys.platform or "freebsd" in sys.platform: TIOCGPGRP = 0x40047477 else: skip("don't know how to test ioctl() on this platform") Modified: pypy/branch/out-of-line-guards/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/imp/importing.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/imp/importing.py Thu Dec 9 16:33:59 2010 @@ -76,7 +76,7 @@ return SEARCH_ERROR, None, None -if sys.platform in ['linux2', 'freebsd']: +if sys.platform == 'linux2' or 'freebsd' in sys.platform: def case_ok(filename): return True else: @@ -529,7 +529,7 @@ space.sys.setmodule(w_module) raise finally: - space.reloading_modules.clear() + del space.reloading_modules[modulename] # __________________________________________________________________ Modified: pypy/branch/out-of-line-guards/pypy/module/itertools/interp_itertools.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/itertools/interp_itertools.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/itertools/interp_itertools.py Thu Dec 9 16:33:59 2010 @@ -391,7 +391,8 @@ self.iterators_w = iterators_w self.current_iterator = 0 self.num_iterators = len(iterators_w) - self.started = False + if self.num_iterators > 0: + self.w_it = iterators_w[0] def iter_w(self): return self.space.wrap(self) @@ -399,26 +400,23 @@ def next_w(self): if self.current_iterator >= self.num_iterators: raise OperationError(self.space.w_StopIteration, self.space.w_None) - if not self.started: - self.current_iterator = 0 - self.w_it = self.iterators_w[self.current_iterator] - self.started = True + try: + return self.space.next(self.w_it) + except OperationError, e: + return self._handle_error(e) + def _handle_error(self, e): while True: + if not e.match(self.space, self.space.w_StopIteration): + raise e + self.current_iterator += 1 + if self.current_iterator >= self.num_iterators: + raise e + self.w_it = self.iterators_w[self.current_iterator] try: - w_obj = self.space.next(self.w_it) + return self.space.next(self.w_it) except OperationError, e: - if e.match(self.space, self.space.w_StopIteration): - self.current_iterator += 1 - if self.current_iterator >= self.num_iterators: - raise OperationError(self.space.w_StopIteration, self.space.w_None) - else: - self.w_it = self.iterators_w[self.current_iterator] - else: - raise - else: - break - return w_obj + pass # loop back to the start of _handle_error(e) def W_Chain___new__(space, w_subtype, args_w): return space.wrap(W_Chain(space, args_w)) @@ -446,8 +444,10 @@ def __init__(self, space, w_fun, args_w): self.space = space - self.identity_fun = (self.space.is_w(w_fun, space.w_None)) - self.w_fun = w_fun + if self.space.is_w(w_fun, space.w_None): + self.w_fun = None + else: + self.w_fun = w_fun iterators_w = [] i = 0 @@ -470,12 +470,26 @@ return self.space.wrap(self) def next_w(self): - w_objects = self.space.newtuple([self.space.next(w_it) for w_it in self.iterators_w]) - if self.identity_fun: + # common case: 1 or 2 arguments + iterators_w = self.iterators_w + length = len(iterators_w) + if length == 1: + objects = [self.space.next(iterators_w[0])] + elif length == 2: + objects = [self.space.next(iterators_w[0]), + self.space.next(iterators_w[1])] + else: + objects = self._get_objects() + w_objects = self.space.newtuple(objects) + if self.w_fun is None: return w_objects else: return self.space.call(self.w_fun, w_objects) + def _get_objects(self): + # the loop is out of the way of the JIT + return [self.space.next(w_elem) for w_elem in self.iterators_w] + def W_IMap___new__(space, w_subtype, w_fun, args_w): if len(args_w) == 0: @@ -769,15 +783,7 @@ raise OperationError(self.space.w_StopIteration, self.space.w_None) if not self.new_group: - # Consume unwanted input until we reach the next group - try: - while True: - self.group_next(self.index) - - except StopIteration: - pass - if self.exhausted: - raise OperationError(self.space.w_StopIteration, self.space.w_None) + self._consume_unwanted_input() if not self.started: self.started = True @@ -799,6 +805,16 @@ w_iterator = self.space.wrap(W_GroupByIterator(self.space, self.index, self)) return self.space.newtuple([self.w_key, w_iterator]) + def _consume_unwanted_input(self): + # Consume unwanted input until we reach the next group + try: + while True: + self.group_next(self.index) + except StopIteration: + pass + if self.exhausted: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + def group_next(self, group_index): if group_index < self.index: raise StopIteration Modified: pypy/branch/out-of-line-guards/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/posix/__init__.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/posix/__init__.py Thu Dec 9 16:33:59 2010 @@ -96,6 +96,8 @@ interpleveldefs['fork'] = 'interp_posix.fork' if hasattr(os, 'openpty'): interpleveldefs['openpty'] = 'interp_posix.openpty' + if hasattr(os, 'forkpty'): + interpleveldefs['forkpty'] = 'interp_posix.forkpty' if hasattr(os, 'waitpid'): interpleveldefs['waitpid'] = 'interp_posix.waitpid' if hasattr(os, 'execv'): Modified: pypy/branch/out-of-line-guards/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/posix/interp_posix.py Thu Dec 9 16:33:59 2010 @@ -608,6 +608,14 @@ raise wrap_oserror(space, e) return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)]) +def forkpty(space): + try: + pid, master_fd = os.forkpty() + except OSError, e: + raise wrap_oserror(space, e) + return space.newtuple([space.wrap(pid), + space.wrap(master_fd)]) + def waitpid(space, pid, options): """ waitpid(pid, options) -> (pid, status) Modified: pypy/branch/out-of-line-guards/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/posix/test/test_posix2.py Thu Dec 9 16:33:59 2010 @@ -331,6 +331,22 @@ data = os.read(master_fd, 100) assert data.startswith('x') + if hasattr(__import__(os.name), "forkpty"): + def test_forkpty(self): + import sys + os = self.posix + childpid, master_fd = os.forkpty() + assert isinstance(childpid, int) + assert isinstance(master_fd, int) + if childpid == 0: + data = os.read(0, 100) + if data.startswith('abc'): + os._exit(42) + else: + os._exit(43) + os.write(master_fd, 'abc\n') + _, status = os.waitpid(childpid, 0) + assert status >> 8 == 42 if hasattr(__import__(os.name), "execv"): def test_execv(self): Modified: pypy/branch/out-of-line-guards/pypy/module/pyexpat/interp_pyexpat.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/pyexpat/interp_pyexpat.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/pyexpat/interp_pyexpat.py Thu Dec 9 16:33:59 2010 @@ -329,7 +329,7 @@ if self.returns_unicode: return space.call_function( space.getattr(space.wrap(s), space.wrap("decode")), - space.wrap(self.encoding), + space.wrap("utf-8"), space.wrap("strict")) else: return space.wrap(s) Modified: pypy/branch/out-of-line-guards/pypy/module/pyexpat/test/test_parser.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/pyexpat/test/test_parser.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/pyexpat/test/test_parser.py Thu Dec 9 16:33:59 2010 @@ -15,3 +15,17 @@ assert res == 1 raises(pyexpat.ExpatError, p.Parse, "3") + + def test_encoding(self): + import pyexpat + for encoding_arg in (None, 'utf-8', 'iso-8859-1'): + for namespace_arg in (None, '{'): + print encoding_arg, namespace_arg + p = pyexpat.ParserCreate(encoding_arg, namespace_arg) + data = [] + p.CharacterDataHandler = lambda s: data.append(s) + encoding = encoding_arg is None and 'utf-8' or encoding_arg + + res = p.Parse(u"\u00f6".encode(encoding), isfinal=True) + assert res == 1 + assert data == [u"\u00f6"] Modified: pypy/branch/out-of-line-guards/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/pypyjit/policy.py Thu Dec 9 16:33:59 2010 @@ -12,19 +12,13 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys', 'array', '_ffi']: + 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator']: return True return False def look_inside_function(self, func): - # this function should never actually return True directly - # but instead call the base implementation mod = func.__module__ or '?' - if mod.startswith('pypy.objspace.'): - # gc_id operation - if func.__name__ == 'id__ANY': - return False if mod == 'pypy.rlib.rbigint' or mod == 'pypy.rlib.rlocale': return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff Modified: pypy/branch/out-of-line-guards/pypy/module/pypyjit/test/test_policy.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/pypyjit/test/test_policy.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/pypyjit/test/test_policy.py Thu Dec 9 16:33:59 2010 @@ -4,7 +4,7 @@ def test_id_any(): from pypy.objspace.std.default import id__ANY - assert not pypypolicy.look_inside_function(id__ANY) + assert pypypolicy.look_inside_function(id__ANY) def test_bigint(): from pypy.rlib.rbigint import rbigint Modified: pypy/branch/out-of-line-guards/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/pypyjit/test/test_pypy_c.py Thu Dec 9 16:33:59 2010 @@ -630,6 +630,32 @@ ''', 3000, ([0], 2000*3)) assert len(self.loops) == 1 + def test_getattr_with_dynamic_attribute(self): + self.run_source(''' + class A(object): + pass + + l = ["x", "y"] + + def main(arg): + sum = 0 + a = A() + a.a1 = 0 + a.a2 = 0 + a.a3 = 0 + a.a4 = 0 + a.a5 = 0 # workaround, because the first five attributes need a promotion + a.x = 1 + a.y = 2 + i = 0 + while i < 2000: + name = l[i % 2] + sum += getattr(a, name) + i += 1 + return sum + ''', 3000, ([0], 3000)) + assert len(self.loops) == 1 + def test_blockstack_virtualizable(self): self.run_source(''' from pypyjit import residual_call @@ -673,11 +699,9 @@ i = t2[3] del t2 return i - ''', 100, ([], 100)) + ''', 40, ([], 100)) bytecode, = self.get_by_bytecode('BINARY_SUBSCR') - assert len(bytecode.get_opnames('new_array')) == 1 - # XXX I would like here to say that it's 0, but unfortunately - # call that can raise is not exchanged into getarrayitem_gc + assert len(bytecode.get_opnames('new_array')) == 0 def test_overflow_checking(self): startvalue = sys.maxint - 2147483647 @@ -861,11 +885,14 @@ def test_array_sum(self): for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): res = 19352859 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array @@ -913,11 +940,14 @@ print '='*65 print '='*20, 'running test for tc=%r' % (tc,), '='*20 res = 73574560 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array Modified: pypy/branch/out-of-line-guards/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/sys/__init__.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/sys/__init__.py Thu Dec 9 16:33:59 2010 @@ -63,8 +63,6 @@ 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', 'hexversion' : 'version.get_hexversion(space)', - 'ps1' : 'space.wrap(">>>> ")', - 'ps2' : 'space.wrap(".... ")', 'displayhook' : 'hook.displayhook', '__displayhook__' : 'hook.__displayhook__', Modified: pypy/branch/out-of-line-guards/pypy/module/sys/state.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/sys/state.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/sys/state.py Thu Dec 9 16:33:59 2010 @@ -33,6 +33,8 @@ raise OSError(errno.ENOTDIR, path) +platform = sys.platform + def getinitialpath(prefix): from pypy.module.sys.version import CPYTHON_VERSION dirname = '%d.%d.%d' % (CPYTHON_VERSION[0], @@ -51,6 +53,15 @@ importlist.append(lib_pypy) importlist.append(python_std_lib_modified) importlist.append(python_std_lib) + # + # List here the extra platform-specific paths. + if platform != 'win32': + importlist.append(os.path.join(python_std_lib, 'plat-'+platform)) + if platform == 'darwin': + platmac = os.path.join(python_std_lib, 'plat-mac') + importlist.append(platmac) + importlist.append(os.path.join(platmac, 'lib-scriptpackages')) + # return importlist def pypy_initial_path(space, srcdir): Modified: pypy/branch/out-of-line-guards/pypy/module/sys/test/test_initialpath.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/sys/test/test_initialpath.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/sys/test/test_initialpath.py Thu Dec 9 16:33:59 2010 @@ -15,4 +15,5 @@ def test_stdlib_in_prefix(tmpdir): dirs = build_hierarchy(tmpdir) path = getinitialpath(str(tmpdir)) - assert path == map(str, dirs) + # we get at least 'dirs', and maybe more (e.g. plat-linux2) + assert path[:len(dirs)] == map(str, dirs) Modified: pypy/branch/out-of-line-guards/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/sys/vm.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/sys/vm.py Thu Dec 9 16:33:59 2010 @@ -42,28 +42,22 @@ f = ec.getnextframe_nohidden(f) return space.wrap(f) -# directly from the C code in ceval.c, might be moved somewhere else. - @jit.dont_look_inside def setrecursionlimit(space, w_new_limit): - """Set the maximum depth of the Python interpreter stack to n. This -limit prevents infinite recursion from causing an overflow of the C -stack and crashing Python. The highest possible limit is platform -dependent.""" + """DEPRECATED on PyPy. Will issue warning and not work + """ new_limit = space.int_w(w_new_limit) if new_limit <= 0: raise OperationError(space.w_ValueError, space.wrap("recursion limit must be positive")) # global recursion_limit # we need to do it without writing globals. + space.warn('setrecursionlimit deprecated', space.w_DeprecationWarning) space.sys.recursionlimit = new_limit def getrecursionlimit(space): - """Return the current value of the recursion limit, the maximum depth - of the Python interpreter stack. This limit prevents infinite - recursion from causing an overflow of the C stack and crashing Python. + """DEPRECATED on PyPy. Will issue warning and not work """ - return space.wrap(space.sys.recursionlimit) def setcheckinterval(space, interval): Modified: pypy/branch/out-of-line-guards/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py Thu Dec 9 16:33:59 2010 @@ -11,21 +11,23 @@ py.test.skip("pypy white-box test") from _ctypes.function import CFuncPtr - guess = CFuncPtr._guess_argtypes + def guess(value): + cobj = CFuncPtr._conv_param(None, value, 0) + return type(cobj) - assert guess([13]) == [c_int] - assert guess([0]) == [c_int] - assert guess(['xca']) == [c_char_p] - assert guess([None]) == [c_void_p] - assert guess([c_int(3)]) == [c_int] - assert guess([u'xca']) == [c_wchar_p] + assert guess(13) == c_int + assert guess(0) == c_int + assert guess('xca') == c_char_p + assert guess(None) == c_void_p + assert guess(c_int(3)) == c_int + assert guess(u'xca') == c_wchar_p class Stuff: pass s = Stuff() s._as_parameter_ = None - assert guess([s]) == [c_void_p] + assert guess(s) == c_void_p def test_guess_unicode(): if not hasattr(sys, 'pypy_translation_info') and sys.platform != 'win32': Modified: pypy/branch/out-of-line-guards/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py (original) +++ pypy/branch/out-of-line-guards/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py Thu Dec 9 16:33:59 2010 @@ -99,7 +99,7 @@ def test_primitive(self): if not hasattr(sys, 'pypy_translation_info'): py.test.skip("pypy white-box test") - assert c_char_p("abc")._objects['0']._buffer[0] == "a" + assert c_char_p("abc")._objects._buffer[0] == "a" assert c_int(3)._objects is None def test_pointer_to_pointer(self): @@ -123,7 +123,7 @@ pass cf = CFUNCTYPE(c_int, c_int)(f) p1 = cast(cf, c_void_p) - assert p1._objects == {'1': cf, '0': {'0': cf}} + assert p1._objects == {id(cf): cf, '0': cf} def test_array_of_struct_with_pointer(self): class S(Structure): @@ -221,7 +221,7 @@ import gc; gc.collect() print 'x =', repr(x) assert x.value == 'hellohello' - assert x._objects.keys() == ['0'] + assert x._objects == 'hellohello' # class datum(Structure): _fields_ = [ Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/complexobject.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/complexobject.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/complexobject.py Thu Dec 9 16:33:59 2010 @@ -23,90 +23,92 @@ """ representation for debugging purposes """ return "" % (w_self.realval, w_self.imagval) -registerimplementation(W_ComplexObject) - -c_1 = (1.0, 0.0) + def sub(self, other): + return W_ComplexObject(self.realval - other.realval, + self.imagval - other.imagval) + + def mul(self, other): + r = self.realval * other.realval - self.imagval * other.imagval + i = self.realval * other.imagval + self.imagval * other.realval + return W_ComplexObject(r, i) + + def div(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 < 0: + abs_r2 = - r2 + else: + abs_r2 = r2 + if i2 < 0: + abs_i2 = - i2 + else: + abs_i2 = i2 + if abs_r2 >= abs_i2: + if abs_r2 == 0.0: + raise ZeroDivisionError + else: + ratio = i2 / r2 + denom = r2 + i2 * ratio + rr = (r1 + i1 * ratio) / denom + ir = (i1 - r1 * ratio) / denom + else: + ratio = r2 / i2 + denom = r2 * ratio + i2 + assert i2 != 0.0 + rr = (r1 * ratio + i1) / denom + ir = (i1 * ratio - r1) / denom + return W_ComplexObject(rr,ir) + + def divmod(self, other): + w_div = self.div(other) + div = math.floor(w_div.realval) + w_mod = self.sub( + W_ComplexObject(other.realval * div, other.imagval * div)) + return (W_ComplexObject(div, 0), w_mod) + + def pow(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 == 0.0 and i2 == 0.0: + rr, ir = 1, 0 + elif r1 == 0.0 and i1 == 0.0: + if i2 != 0.0 or r2 < 0.0: + raise ZeroDivisionError + rr, ir = (0.0, 0.0) + else: + vabs = math.hypot(r1,i1) + len = math.pow(vabs,r2) + at = math.atan2(i1,r1) + phase = at * r2 + if i2 != 0.0: + len /= math.exp(at * i2) + phase += i2 * math.log(vabs) + rr = len * math.cos(phase) + ir = len * math.sin(phase) + return W_ComplexObject(rr, ir) + + def pow_int(self, n): + if n > 100 or n < -100: + return self.pow(W_ComplexObject(1.0 * n, 0.0)) + elif n > 0: + return self.pow_positive_int(n) + else: + return w_one.div(self.pow_positive_int(-n)) -def _sum(c1, c2): - return (c1[0]+c2[0],c1[1]+c2[1]) + def pow_positive_int(self, n): + mask = 1 + w_result = w_one + while mask > 0 and n >= mask: + if n & mask: + w_result = w_result.mul(self) + mask <<= 1 + self = self.mul(self) -def _diff(c1, c2): - return (c1[0]-c2[0],c1[1]-c2[1]) + return w_result -def _prod(c1, c2): - r = c1[0]*c2[0] - c1[1]*c2[1] - i = c1[0]*c2[1] + c1[1]*c2[0] - return (r,i) - -def _quot(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 < 0: - abs_r2 = - r2 - else: - abs_r2 = r2 - if i2 < 0: - abs_i2 = - i2 - else: - abs_i2 = i2 - if abs_r2 >= abs_i2: - if abs_r2 == 0.0: - raise ZeroDivisionError - else: - ratio = i2 / r2 - denom = r2 + i2 * ratio - rr = (r1 + i1 * ratio) / denom - ir = (i1 - r1 * ratio) / denom - else: - ratio = r2 / i2 - denom = r2 * ratio + i2 - assert i2 != 0.0 - rr = (r1 * ratio + i1) / denom - ir = (i1 * ratio - r1) / denom - return (rr,ir) - -def _pow(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 == 0.0 and i2 == 0.0: - rr, ir = c_1 - elif r1 == 0.0 and i1 == 0.0: - if i2 != 0.0 or r2 < 0.0: - raise ZeroDivisionError - rr, ir = (0.0, 0.0) - else: - vabs = math.hypot(r1,i1) - len = math.pow(vabs,r2) - at = math.atan2(i1,r1) - phase = at * r2 - if i2 != 0.0: - len /= math.exp(at * i2) - phase += i2 * math.log(vabs) - rr = len * math.cos(phase) - ir = len * math.sin(phase) - return (rr, ir) - -def _powu(c,n): - mask = 1; - rr, ir = c_1 - rp = c[0] - ip = c[1] - while mask > 0 and n >= mask: - if n & mask: - rr, ir = _prod((rr, ir), (rp, ip)) - mask <<= 1 - rp, ip = _prod((rp, ip), (rp, ip)) - - return (rr, ir) - -def _powi(c,n): - if n > 100 or n < -100: - return _pow(c,(1.0 * n, 0.0)) - elif n > 0: - return _powu(c, n) - else: - return _quot(c_1, _powu(c, -n)) +registerimplementation(W_ComplexObject) +w_one = W_ComplexObject(1, 0) def delegate_Bool2Complex(space, w_bool): @@ -126,38 +128,25 @@ return W_ComplexObject(w_float.floatval, 0.0) def hash__Complex(space, w_value): - #this is straight out of CPython complex implementation - hashreal = _hash_float(space, w_value.realval) - if hashreal == -1: - return space.newint(-1) hashimg = _hash_float(space, w_value.imagval) - if hashimg == -1: - return space.newint(-1) combined = hashreal + 1000003 * hashimg - if (combined == -1): - combined = -2 return space.newint(combined) -def _w2t(space, w_complex): - "convert an interplevel complex object to a tuple representation" - return w_complex.realval, w_complex.imagval - -def _t2w(space, c): - return W_ComplexObject(c[0], c[1]) - def add__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _sum(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval + w_complex2.realval, + w_complex1.imagval + w_complex2.imagval) def sub__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _diff(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval - w_complex2.realval, + w_complex1.imagval - w_complex2.imagval) def mul__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _prod(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.mul(w_complex2) def div__Complex_Complex(space, w_complex1, w_complex2): try: - return _t2w(space, _quot(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.div(w_complex2) except ZeroDivisionError, e: raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) @@ -165,49 +154,38 @@ def mod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[1] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex remainder")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - - return _t2w(space, mod) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) def divmod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + div, mod = w_complex1.divmod(w_complex2) except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex divmod()")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - w_div = _t2w(space, div) - w_mod = _t2w(space, mod) - return space.newtuple([w_div, w_mod]) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) + return space.newtuple([div, mod]) def floordiv__Complex_Complex(space, w_complex1, w_complex2): + # don't care about the slight slowdown you get from using divmod try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[0] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex floordiv()")) - div = (math.floor(div[0]), 0.0) - return _t2w(space, div) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) -def pow__Complex_Complex_ANY(space, w_complex1, w_complex2, thirdArg): +def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) + int_exponent = int(w_exponent.realval) try: - v = _w2t(space, w_complex1) - exponent = _w2t(space, w_complex2) - int_exponent = int(exponent[0]) - if exponent[1] == 0.0 and exponent[0] == int_exponent: - p = _powi(v, int_exponent) + if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: + w_p = w_complex.pow_int(int_exponent) else: - p = _pow(v, exponent) + w_p = w_complex.pow(w_exponent) except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("0.0 to a negative or complex power")) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("complex exponentiation")) - return _t2w(space, p) + return w_p def neg__Complex(space, w_complex): return W_ComplexObject(-w_complex.realval, -w_complex.imagval) Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/complextype.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/complextype.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/complextype.py Thu Dec 9 16:33:59 2010 @@ -1,7 +1,7 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.strutil import interp_string_to_float, ParseStringError +from pypy.objspace.std.strutil import string_to_float, ParseStringError from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod @@ -131,8 +131,8 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) try: - realval = interp_string_to_float(space, realstr) - imagval = interp_string_to_float(space, imagstr) + realval = string_to_float(realstr) + imagval = string_to_float(imagstr) except ParseStringError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) else: Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/floattype.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/floattype.py Thu Dec 9 16:33:59 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.strutil import ParseStringError -from pypy.objspace.std.strutil import interp_string_to_float +from pypy.objspace.std.strutil import string_to_float def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject @@ -10,7 +10,7 @@ if space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) @@ -21,7 +21,7 @@ from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/longobject.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/longobject.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/longobject.py Thu Dec 9 16:33:59 2010 @@ -45,19 +45,6 @@ fromrarith_int._annspecialcase_ = "specialize:argtype(0)" fromrarith_int = staticmethod(fromrarith_int) - def fromdecimalstr(s): - return W_LongObject(rbigint.fromdecimalstr(s)) - fromdecimalstr = staticmethod(fromdecimalstr) - - def _count_bits(self): - return self.num._count_bits() - - def is_odd(self): - return self.num.is_odd() - - def get_sign(self): - return self.num.sign - registerimplementation(W_LongObject) # bool-to-long Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/mapdict.py Thu Dec 9 16:33:59 2010 @@ -42,11 +42,23 @@ return None def index(self, selector): - if (self.space.config.objspace.std.withmethodcache and - not jit.we_are_jitted()): - return self._index_cache(selector) + if jit.we_are_jitted(): + # hack for the jit: + # the _index method is pure too, but its argument is never + # constant, because it is always a new tuple + return self._index_jit_pure(selector[0], selector[1]) else: - return self._index(selector) + return self._index_indirection(selector) + + @jit.purefunction + def _index_jit_pure(self, name, index): + return self._index_indirection((name, index)) + + @jit.dont_look_inside + def _index_indirection(self, selector): + if (self.space.config.objspace.std.withmethodcache): + return self._index_cache(selector) + return self._index(selector) @jit.dont_look_inside def _index_cache(self, selector): @@ -498,10 +510,11 @@ def _mapdict_read_storage(self, index): assert index >= 0 - for i in rangenmin1: - if index == i: - erased = getattr(self, "_value%s" % i) - return rerased.unerase(erased, W_Root) + if index < nmin1: + for i in rangenmin1: + if index == i: + erased = getattr(self, "_value%s" % i) + return rerased.unerase(erased, W_Root) if self._has_storage_list(): return self._mapdict_get_storage_list()[index - nmin1] erased = getattr(self, "_value%s" % nmin1) Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/objspace.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/objspace.py Thu Dec 9 16:33:59 2010 @@ -9,7 +9,7 @@ from pypy.objspace.descroperation import DescrOperation, raiseattrerror from pypy.rlib.objectmodel import instantiate, r_dict, specialize from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib.rarithmetic import base_int +from pypy.rlib.rarithmetic import base_int, widen from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.jit import hint from pypy.tool.sourcetools import func_with_new_name @@ -176,7 +176,11 @@ #print 'wrapping', x, '->', w_result return w_result if isinstance(x, base_int): - return W_LongObject.fromrarith_int(x) + x = widen(x) + if isinstance(x, int): + return self.newint(x) + else: + return W_LongObject.fromrarith_int(x) # _____ below here is where the annotator should not get _____ @@ -372,7 +376,7 @@ self, w_obj, expected_length)[:]) if expected_length != -1 and len(t) != expected_length: raise self._wrap_expected_length(expected_length, len(t)) - return t + return make_sure_not_resized(t) def fixedview_unroll(self, w_obj, expected_length=-1): return self.fixedview(w_obj, expected_length, unroll=True) Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/strutil.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/strutil.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/strutil.py Thu Dec 9 16:33:59 2010 @@ -150,7 +150,7 @@ del calc_mantissa_bits MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1 -def interp_string_to_float(space, s): +def string_to_float(s): """ Conversion of string to float. This version tries to only raise on invalid literals. @@ -162,10 +162,9 @@ s = strip_spaces(s) if not s: - raise OperationError(space.w_ValueError, space.wrap( - "empty string for float()")) + raise ParseStringError("empty string for float()") + - low = s.lower() if low == "-inf": return -INFINITY @@ -204,68 +203,56 @@ if len(digits) == 0: digits = '0' - # a few abbreviations - from pypy.objspace.std import longobject - mklong = longobject.W_LongObject.fromint - d2long = longobject.W_LongObject.fromdecimalstr - adlong = longobject.add__Long_Long - longup = longobject.pow__Long_Long_None - multip = longobject.mul__Long_Long - divide = longobject.div__Long_Long - lshift = longobject.lshift__Long_Long - rshift = longobject.rshift__Long_Long - # 4) compute the exponent and truncate to +-400 if not exponent: exponent = '0' - w_le = d2long(exponent) - w_le = adlong(space, w_le, mklong(space, dexp)) + long_exponent = rbigint.fromdecimalstr(exponent) + long_exponent = long_exponent.add(rbigint.fromint(dexp)) try: - e = w_le.toint() + e = long_exponent.toint() except OverflowError: # XXX poking at internals - e = w_le.num.sign * 400 - if e >= 400: - e = 400 - elif e <= -400: - e = -400 + e = long_exponent.sign * 400 + else: + if e >= 400: + e = 400 + elif e <= -400: + e = -400 # 5) compute the value using long math and proper rounding. - w_lr = d2long(digits) - w_10 = mklong(space, 10) - w_1 = mklong(space, 1) + b_digits = rbigint.fromdecimalstr(digits) + b_10 = rbigint.fromint(10) + b_1 = rbigint.fromint(1) if e >= 0: bits = 0 - w_pten = longup(space, w_10, mklong(space, e), space.w_None) - w_m = multip(space, w_lr, w_pten) + b_power_of_ten = b_10.pow(rbigint.fromint(e)) + b_mantissa = b_digits.mul(b_power_of_ten) else: # compute a sufficiently large scale prec = MANTISSA_DIGITS * 2 + 22 # 128, maybe bits = - (int(math.ceil(-e / math.log10(2.0) - 1e-10)) + prec) - w_scale = lshift(space, w_1, mklong(space, -bits)) - w_pten = longup(space, w_10, mklong(space, -e), None) - w_tmp = multip(space, w_lr, w_scale) - w_m = divide(space, w_tmp, w_pten) + b_scale = b_1.lshift(-bits) + b_power_of_ten = b_10.pow(rbigint.fromint(-e)) + b_mantissa = b_digits.mul(b_scale).div(b_power_of_ten) # we now have a fairly large mantissa. # Shift it and round the last bit. # first estimate the bits and do a big shift - mbits = w_m._count_bits() + mbits = b_mantissa._count_bits() needed = MANTISSA_BITS if mbits > needed: if mbits > needed+1: shifted = mbits - (needed+1) - w_m = rshift(space, w_m, mklong(space, shifted)) + b_mantissa = b_mantissa.rshift(shifted) bits += shifted # do the rounding bits += 1 - round = w_m.is_odd() - w_m = rshift(space, w_m, w_1) - w_m = adlong(space, w_m, mklong(space, round)) + round = b_mantissa.is_odd() + b_mantissa = b_mantissa.rshift(1).add(rbigint.fromint(round)) try: - r = math.ldexp(w_m.tofloat(), bits) + r = math.ldexp(b_mantissa.tofloat(), bits) # XXX I guess we do not check for overflow in ldexp as we agreed to! if r == 2*r and r != 0.0: raise OverflowError Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/test/test_complexobject.py Thu Dec 9 16:33:59 2010 @@ -1,5 +1,6 @@ import py -from pypy.objspace.std import complexobject as cobj +from pypy.objspace.std.complexobject import W_ComplexObject, \ + pow__Complex_Complex_ANY from pypy.objspace.std import complextype as cobjtype from pypy.objspace.std.multimethod import FailedToImplement from pypy.objspace.std.stringobject import W_StringObject @@ -11,7 +12,7 @@ def _test_instantiation(self): def _t_complex(r=0.0,i=0.0): - c = cobj.W_ComplexObject(r, i) + c = W_ComplexObject(r, i) assert c.real == float(r) and c.imag == float(i) pairs = ( (1, 1), @@ -38,21 +39,31 @@ test_cparse('.e+5', '.e+5', '0.0') def test_pow(self): - assert cobj._pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) - assert cobj._pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) - rr, ir = cobj._pow((0.0,1.0),(2.0,0.0)) + def _pow((r1, i1), (r2, i2)): + w_res = W_ComplexObject(r1, i1).pow(W_ComplexObject(r2, i2)) + return w_res.realval, w_res.imagval + assert _pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) + assert _pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) + rr, ir = _pow((0.0,1.0),(2.0,0.0)) assert abs(-1.0 - rr) < EPS assert abs(0.0 - ir) < EPS - assert cobj._powu((0.0,2.0),0) == (1.0,0.0) - assert cobj._powu((0.0,0.0),2) == (0.0,0.0) - assert cobj._powu((0.0,1.0),2) == (-1.0,0.0) - assert cobj._powi((0.0,2.0),0) == (1.0,0.0) - assert cobj._powi((0.0,0.0),2) == (0.0,0.0) - assert cobj._powi((0.0,1.0),2) == (-1.0,0.0) - c = cobj.W_ComplexObject(0.0,1.0) - p = cobj.W_ComplexObject(2.0,0.0) - r = cobj.pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) + def _powu((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_positive_int(n) + return w_res.realval, w_res.imagval + assert _powu((0.0,2.0),0) == (1.0,0.0) + assert _powu((0.0,0.0),2) == (0.0,0.0) + assert _powu((0.0,1.0),2) == (-1.0,0.0) + + def _powi((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_int(n) + return w_res.realval, w_res.imagval + assert _powi((0.0,2.0),0) == (1.0,0.0) + assert _powi((0.0,0.0),2) == (0.0,0.0) + assert _powi((0.0,1.0),2) == (-1.0,0.0) + c = W_ComplexObject(0.0,1.0) + p = W_ComplexObject(2.0,0.0) + r = pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) assert r.realval == -1.0 assert r.imagval == 0.0 Modified: pypy/branch/out-of-line-guards/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/branch/out-of-line-guards/pypy/objspace/std/test/test_strutil.py Thu Dec 9 16:33:59 2010 @@ -131,8 +131,6 @@ assert string_to_bigint('1891234174197319').tolong() == 1891234174197319 def test_string_to_float(self): - def string_to_float(x): - return interp_string_to_float(self.space, x) assert string_to_float('0') == 0.0 assert string_to_float('1') == 1.0 assert string_to_float('-1.5') == -1.5 @@ -180,3 +178,4 @@ print repr(s) if s.strip(): # empty s raises OperationError directly py.test.raises(ParseStringError, string_to_float, s) + py.test.raises(ParseStringError, string_to_float, "") Modified: pypy/branch/out-of-line-guards/pypy/rlib/debug.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/debug.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/debug.py Thu Dec 9 16:33:59 2010 @@ -53,13 +53,11 @@ _log = None # patched from tests to be an object of class DebugLog # or compatible -_stderr = sys.stderr # alternatively, this is patched from tests - # (redirects debug_print(), but not debug_start/stop) def debug_print(*args): for arg in args: - print >> _stderr, arg, - print >> _stderr + print >> sys.stderr, arg, + print >> sys.stderr if _log is not None: _log.debug_print(*args) @@ -87,13 +85,15 @@ _stop_colors = "" def debug_start(category): - print >> sys.stderr, '%s[%s] {%s%s' % (_start_colors_1, time.clock(), + c = int(time.clock() * 100) + print >> sys.stderr, '%s[%x] {%s%s' % (_start_colors_1, c, category, _stop_colors) if _log is not None: _log.debug_start(category) def debug_stop(category): - print >> sys.stderr, '%s[%s] %s}%s' % (_start_colors_2, time.clock(), + c = int(time.clock() * 100) + print >> sys.stderr, '%s[%x] %s}%s' % (_start_colors_2, c, category, _stop_colors) if _log is not None: _log.debug_stop(category) @@ -226,31 +226,6 @@ hop.exception_cannot_occur() return hop.inputarg(hop.args_r[0], arg=0) -def make_sure_not_modified(arg): - """ Function checking whether annotation of SomeList is never resized - and never modified, useful for debugging. Does nothing when run directly - """ - return arg - -class Entry(ExtRegistryEntry): - _about_ = make_sure_not_modified - - def compute_result_annotation(self, s_arg): - from pypy.annotation.model import SomeList - assert isinstance(s_arg, SomeList) - # the logic behind it is that we try not to propagate - # make_sure_not_resized, when list comprehension is not on - if self.bookkeeper.annotator.translator.config.translation.list_comprehension_operations: - s_arg.listdef.never_mutate() - else: - from pypy.annotation.annrpython import log - log.WARNING('make_sure_not_modified called, but has no effect since list_comprehension is off') - return s_arg - - def specialize_call(self, hop): - hop.exception_cannot_occur() - return hop.inputarg(hop.args_r[0], arg=0) - class IntegerCanBeNegative(Exception): pass Modified: pypy/branch/out-of-line-guards/pypy/rlib/jit.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/jit.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/jit.py Thu Dec 9 16:33:59 2010 @@ -156,7 +156,7 @@ def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1, arg3=-sys.maxint-1, arg4=-sys.maxint-1): - """When JITted, cause an extra operation DEBUG_MERGE_POINT to appear in + """When JITted, cause an extra operation JIT_DEBUG to appear in the graphs. Should not be left after debugging.""" keepalive_until_here(string) # otherwise the whole function call is removed jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)' @@ -260,17 +260,12 @@ OPTIMIZER_NO_PERFECTSPEC = 1 OPTIMIZER_FULL = 2 -DEBUG_OFF = 0 -DEBUG_PROFILE = 1 -DEBUG_STEPS = 2 -DEBUG_DETAILED = 3 - PARAMETERS = {'threshold': 1000, 'trace_eagerness': 200, 'trace_limit': 10000, 'inlining': False, 'optimizer': OPTIMIZER_FULL, - 'debug' : DEBUG_STEPS, + 'loop_longevity': 1000, } unroll_parameters = unrolling_iterable(PARAMETERS.keys()) Modified: pypy/branch/out-of-line-guards/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/libffi.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/libffi.py Thu Dec 9 16:33:59 2010 @@ -178,6 +178,9 @@ # the optimizer will fail to recognize the pattern and won't turn it # into a fast CALL. Note that "arg = arg.next" is optimized away, # assuming that archain is completely virtual. + if argchain.numargs != len(self.argtypes): + raise TypeError, 'Wrong number of arguments: %d expected, got %d' %\ + (argchain.numargs, len(self.argtypes)) ll_args = self._prepare() i = 0 arg = argchain.first Modified: pypy/branch/out-of-line-guards/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/rarithmetic.py Thu Dec 9 16:33:59 2010 @@ -92,7 +92,8 @@ return False r_class = rffi.platform.numbertype_to_rclass[tp] assert issubclass(r_class, base_int) - return r_class.BITS < LONG_BIT + return r_class.BITS < LONG_BIT or ( + r_class.BITS == LONG_BIT and r_class.SIGNED) _should_widen_type._annspecialcase_ = 'specialize:memo' del _bits, _itest, _Ltest @@ -389,6 +390,11 @@ r_longlong = build_int('r_longlong', True, 64) r_ulonglong = build_int('r_ulonglong', False, 64) +if r_longlong is not r_int: + r_int64 = r_longlong +else: + r_int64 = int + # float as string -> sign, beforept, afterpt, exponent Modified: pypy/branch/out-of-line-guards/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/rdynload.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/rdynload.py Thu Dec 9 16:33:59 2010 @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD_7 = platform.name == "freebsd7" +_FREEBSD = platform.name == "freebsd" if _WIN32: from pypy.rlib import rwin32 @@ -27,7 +27,7 @@ else: pre_include_bits = [] -if _FREEBSD_7 or _WIN32: +if _FREEBSD or _WIN32: libraries = [] else: libraries = ['dl'] Modified: pypy/branch/out-of-line-guards/pypy/rlib/rerased.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/rerased.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/rerased.py Thu Dec 9 16:33:59 2010 @@ -91,7 +91,7 @@ return annmodel.SomeInteger() assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() return annmodel.SomeInstance(clsdef) def specialize_call(self, hop): @@ -108,7 +108,7 @@ def compute_result_annotation(self, s_obj, s_type): assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() s_item = annmodel.SomeInstance(clsdef) return self.bookkeeper.newlist(s_item) Modified: pypy/branch/out-of-line-guards/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/rmmap.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/rmmap.py Thu Dec 9 16:33:59 2010 @@ -67,7 +67,7 @@ constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE', - 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE'] + 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS'] for name in constant_names: setattr(CConfig, name, rffi_platform.ConstantInteger(name)) Modified: pypy/branch/out-of-line-guards/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/rsre/rsre_core.py Thu Dec 9 16:33:59 2010 @@ -1,5 +1,5 @@ import sys -from pypy.rlib.debug import check_nonneg, make_sure_not_modified +from pypy.rlib.debug import check_nonneg from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rsre import rsre_char from pypy.tool.sourcetools import func_with_new_name @@ -471,7 +471,6 @@ while True: op = ctx.pat(ppos) ppos += 1 - make_sure_not_modified(ctx.pattern) #jit.jit_debug("sre_match", op, ppos, ptr) # Modified: pypy/branch/out-of-line-guards/pypy/rlib/rsre/test/test_zjit.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/rsre/test/test_zjit.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/rsre/test/test_zjit.py Thu Dec 9 16:33:59 2010 @@ -1,6 +1,5 @@ from pypy.jit.metainterp.test import test_basic from pypy.rlib.nonconst import NonConstant -from pypy.rlib.debug import make_sure_not_modified from pypy.rlib.rsre.test.test_match import get_code from pypy.rlib.rsre import rsre_core from pypy.rpython.lltypesystem import lltype Modified: pypy/branch/out-of-line-guards/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/test/test_debug.py Thu Dec 9 16:33:59 2010 @@ -42,14 +42,14 @@ py.test.raises(IntegerCanBeNegative, interpret, g, [9]) def test_make_sure_not_resized(): - from pypy.annotation.listdef import TooLateForChange + from pypy.annotation.listdef import ListChangeUnallowed def f(): result = [1,2,3] make_sure_not_resized(result) result.append(4) return len(result) - py.test.raises(TooLateForChange, interpret, f, [], + py.test.raises(ListChangeUnallowed, interpret, f, [], list_comprehension_operations=True) Modified: pypy/branch/out-of-line-guards/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rlib/test/test_libffi.py (original) +++ pypy/branch/out-of-line-guards/pypy/rlib/test/test_libffi.py Thu Dec 9 16:33:59 2010 @@ -262,3 +262,24 @@ # res = self.call(get_dummy, [], rffi.LONG) assert res == initval+1 + + def test_wrong_number_of_arguments(self): + from pypy.rpython.llinterp import LLException + libfoo = self.get_libfoo() + func = (libfoo, 'sum_xy', [types.sint, types.double], types.sint) + + glob = globals() + loc = locals() + def my_raises(s): + try: + exec s in glob, loc + except TypeError: + pass + except LLException, e: + if str(e) != "": + raise + else: + assert False, 'Did not raise' + + my_raises("self.call(func, [38], rffi.LONG)") # one less + my_raises("self.call(func, [38, 12.3, 42], rffi.LONG)") # one more Modified: pypy/branch/out-of-line-guards/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/extfunc.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/extfunc.py Thu Dec 9 16:33:59 2010 @@ -201,6 +201,11 @@ exec py.code.compile(""" from pypy.rlib.objectmodel import running_on_llinterp from pypy.rlib.debug import llinterpcall + from pypy.rlib.jit import dont_look_inside + # note: we say 'dont_look_inside' mostly because the + # JIT does not support 'running_on_llinterp', but in + # theory it is probably right to stop jitting anyway. + @dont_look_inside def ll_wrapper(%s): if running_on_llinterp: return llinterpcall(s_result, fakeimpl, %s) Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/ll2ctypes.py Thu Dec 9 16:33:59 2010 @@ -409,6 +409,7 @@ subcls = get_common_subclass(mixin_cls, instance.__class__) instance.__class__ = subcls instance._storage = ctypes_storage + assert ctypes_storage # null pointer? class _parentable_mixin(object): """Mixin added to _parentable containers when they become ctypes-based. @@ -442,6 +443,9 @@ "not allocated from RPython at all") self._storage = None + def _getid(self): + return self._addressof_storage() + def __eq__(self, other): if isinstance(other, _llgcopaque): addressof_other = other.intval @@ -1291,7 +1295,7 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7'): + elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/llarena.py Thu Dec 9 16:33:59 2010 @@ -26,6 +26,7 @@ self.objectptrs = {} # {offset: ptr-to-container} self.objectsizes = {} # {offset: size} self.freed = False + self.protect_inaccessible = None self.reset(zero) def __repr__(self): @@ -59,6 +60,8 @@ def check(self): if self.freed: raise ArenaError("arena was already freed") + if self.protect_inaccessible is not None: + raise ArenaError("arena is currently arena_protect()ed") def _getid(self): address, length = self.usagemap.buffer_info() @@ -127,6 +130,21 @@ def mark_freed(self): self.freed = True # this method is a hook for tests + def set_protect(self, inaccessible): + if inaccessible: + assert self.protect_inaccessible is None + saved = [] + for ptr in self.objectptrs.values(): + obj = ptr._obj + saved.append((obj, obj._protect())) + self.protect_inaccessible = saved + else: + assert self.protect_inaccessible is not None + saved = self.protect_inaccessible + for obj, storage in saved: + obj._unprotect(storage) + self.protect_inaccessible = None + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -365,6 +383,16 @@ """ return Arena(ptr.arena.nbytes, False).getaddr(0) +def arena_protect(arena_addr, size, inaccessible): + """For debugging, set or reset memory protection on an arena. + For now, the starting point and size should reference the whole arena. + The value of 'inaccessible' is a boolean. + """ + arena_addr = getfakearenaaddress(arena_addr) + assert arena_addr.offset == 0 + assert size == arena_addr.arena.nbytes + arena_addr.arena.set_protect(inaccessible) + # ____________________________________________________________ # # Translation support: the functions above turn into the code below. @@ -475,6 +503,44 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear +if os.name == "posix": + from pypy.translator.tool.cbuild import ExternalCompilationInfo + _eci = ExternalCompilationInfo(includes=['sys/mman.h']) + raw_mprotect = rffi.llexternal('mprotect', + [llmemory.Address, rffi.SIZE_T, rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True, + compilation_info=_eci) + def llimpl_protect(addr, size, inaccessible): + if inaccessible: + prot = 0 + else: + from pypy.rlib.rmmap import PROT_READ, PROT_WRITE + prot = PROT_READ | PROT_WRITE + raw_mprotect(addr, rffi.cast(rffi.SIZE_T, size), + rffi.cast(rffi.INT, prot)) + # ignore potential errors + has_protect = True + +elif os.name == 'nt': + def llimpl_protect(addr, size, inaccessible): + from pypy.rlib.rmmap import VirtualProtect, LPDWORD + if inaccessible: + from pypy.rlib.rmmap import PAGE_NOACCESS as newprotect + else: + from pypy.rlib.rmmap import PAGE_READWRITE as newprotect + arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') + VirtualProtect(rffi.cast(rffi.VOIDP, addr), + rffi.cast(rffi.SIZE_T, size), + newprotect, + arg) + # ignore potential errors + lltype.free(arg, flavor='raw') + has_protect = True + +else: + has_protect = False + llimpl_malloc = rffi.llexternal('malloc', [lltype.Signed], llmemory.Address, sandboxsafe=True, _nowrapper=True) @@ -544,6 +610,21 @@ 'll_arena.arena_new_view', llimpl=llimpl_arena_new_view, llfakeimpl=arena_new_view, sandboxsafe=True) +def llimpl_arena_protect(addr, size, inaccessible): + if has_protect: + # do some alignment + start = rffi.cast(lltype.Signed, addr) + end = start + size + start = (start + 4095) & ~ 4095 + end = end & ~ 4095 + if end > start: + llimpl_protect(rffi.cast(llmemory.Address, start), end-start, + inaccessible) +register_external(arena_protect, [llmemory.Address, lltype.Signed, + lltype.Bool], lltype.Void, + 'll_arena.arena_protect', llimpl=llimpl_arena_protect, + llfakeimpl=arena_protect, sandboxsafe=True) + def llimpl_getfakearenaaddress(addr): return addr register_external(getfakearenaaddress, [llmemory.Address], llmemory.Address, Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/llmemory.py Thu Dec 9 16:33:59 2010 @@ -93,8 +93,10 @@ return endmarker._as_ptr() else: return parent.getitem(index)._as_ptr() - elif (isinstance(A, lltype.FixedSizeArray) and - array_item_type_match(A.OF, self.TYPE)): + elif ((isinstance(A, lltype.FixedSizeArray) + or (isinstance(A, lltype.Array) and A._hints.get('nolength', + False))) + and array_item_type_match(A.OF, self.TYPE)): # for array of primitives or pointers return lltype.direct_ptradd(firstitemptr, self.repeat) else: Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/lltype.py Thu Dec 9 16:33:59 2010 @@ -1381,6 +1381,15 @@ self._check() # no double-frees self._storage = None + def _protect(self): + result = self._storage + self._free() # no double-frees or double-protects + return result + + def _unprotect(self, saved_storage): + assert self._storage is None + self._storage = saved_storage + def _was_freed(self): if self._storage is None: return True Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rbuiltin.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rbuiltin.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rbuiltin.py Thu Dec 9 16:33:59 2010 @@ -42,7 +42,7 @@ return hop.genop('cast_pointer', [v_inst], # v_type implicit in r_result resulttype = hop.r_result.lowleveltype) - classdef = s_class.descriptions.keys()[0].getuniqueclassdef() + classdef = s_class.any_description().getuniqueclassdef() return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops) def rtype_builtin_hasattr(hop): Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py Thu Dec 9 16:33:59 2010 @@ -410,7 +410,7 @@ source_classdef = source_desc.getclassdef(None) source_repr = getinstancerepr(self.rtyper, source_classdef) assert len(s_func.descriptions) == 1 - funcdesc = s_func.descriptions.keys()[0] + funcdesc, = s_func.descriptions graph = funcdesc.getuniquegraph() FUNCTYPE = FuncType([Ptr(source_repr.object_type)], Void) destrptr = functionptr(FUNCTYPE, graph.name, Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rdict.py Thu Dec 9 16:33:59 2010 @@ -581,7 +581,7 @@ def ll_dict_lookup_clean(d, hash): # a simplified version of ll_dict_lookup() which assumes that the # key is new, and the dictionary doesn't contain deleted entries. - # It only find the next free slot for the given hash. + # It only finds the next free slot for the given hash. entries = d.entries mask = len(entries) - 1 i = hash & mask Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rpbc.py Thu Dec 9 16:33:59 2010 @@ -127,7 +127,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(self.rtyper, self.callfamily) assert len(uniquerows) == 1 @@ -166,7 +166,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) def convert_desc(self, funcdesc): @@ -230,7 +230,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/test/test_llarena.py Thu Dec 9 16:33:59 2010 @@ -6,6 +6,8 @@ from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view from pypy.rpython.lltypesystem.llarena import arena_shrink_obj +from pypy.rpython.lltypesystem.llarena import arena_protect, has_protect +from pypy.translator.c.test import test_genc, test_standalone def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -265,8 +267,7 @@ assert res == 42 def test_compiled(): - from pypy.translator.c.test.test_genc import compile - fn = compile(test_look_inside_object, []) + fn = test_genc.compile(test_look_inside_object, []) res = fn() assert res == 42 @@ -282,3 +283,51 @@ arena_reserve(a, size_gc_header + llmemory.sizeof(S, 10)) arena_shrink_obj(a, size_gc_header + llmemory.sizeof(S, 5)) arena_reset(a, size_gc_header + llmemory.sizeof(S, 5), False) + +def test_arena_protect(): + a = arena_malloc(100, False) + S = lltype.Struct('S', ('x', lltype.Signed)) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 100, True) + py.test.raises(ArenaError, arena_reserve, a + 48, llmemory.sizeof(S)) + py.test.raises(RuntimeError, "p.x") + py.test.raises(RuntimeError, "p.x = 124") + arena_protect(a, 100, False) + assert p.x == 123 + p.x = 125 + assert p.x == 125 + + +class TestStandalone(test_standalone.StandaloneTests): + def test_compiled_arena_protect(self): + import os + from pypy.translator.c.test.test_genc import compile + S = lltype.Struct('S', ('x', lltype.Signed)) + # + def fn(argv): + testrun = int(argv[1]) + a = arena_malloc(65536, False) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a + 23432, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 65536, True) + result = 0 + if testrun == 1: + print p.x # segfault + if testrun == 2: + p.x = 124 # segfault + arena_protect(a, 65536, False) + p.x += 10 + print p.x + return 0 + # + t, cbuilder = self.compile(fn) + data = cbuilder.cmdexec('0') + assert data == '133\n' + if has_protect: + cbuilder.cmdexec('1', expect_crash=True) + cbuilder.cmdexec('2', expect_crash=True) Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/base.py Thu Dec 9 16:33:59 2010 @@ -5,7 +5,6 @@ from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rlib.rarithmetic import r_uint TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed), ('size', lltype.Signed), @@ -19,6 +18,7 @@ malloc_zero_filled = False prebuilt_gc_objects_are_static_roots = True object_minimal_size = 0 + gcflag_extra = 0 # or a real GC flag that is always 0 when not collecting def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, translated_to_c=True): @@ -36,6 +36,12 @@ self.finalizer_lock_count = 0 self.run_finalizers = self.AddressDeque() + def post_setup(self): + # More stuff that needs to be initialized when the GC is already + # fully working. (Only called by gctransform/framework for now.) + from pypy.rpython.memory.gc import env + self.DEBUG = env.read_from_env('PYPY_GC_DEBUG') + def _teardown(self): pass @@ -48,7 +54,8 @@ # The following flag enables costly consistency checks after each # collection. It is automatically set to True by test_gc.py. The # checking logic is translatable, so the flag can be set to True - # here before translation. + # here before translation. At run-time, if PYPY_GC_DEBUG is set, + # then it is also set to True. DEBUG = False def set_query_functions(self, is_varsize, has_gcptr_in_varsize, @@ -410,42 +417,6 @@ GCClass = getattr(module, classname) return GCClass, GCClass.TRANSLATION_PARAMS -def _read_float_and_factor_from_env(varname): - import os - value = os.environ.get(varname) - if value: - if len(value) > 1 and value[-1] in 'bB': - value = value[:-1] - realvalue = value[:-1] - if value[-1] in 'kK': - factor = 1024 - elif value[-1] in 'mM': - factor = 1024*1024 - elif value[-1] in 'gG': - factor = 1024*1024*1024 - else: - factor = 1 - realvalue = value - try: - return (float(realvalue), factor) - except ValueError: - pass - return (0.0, 0) - -def read_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - return int(value * factor) - -def read_uint_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - return r_uint(value * factor) - -def read_float_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - if factor != 1: - return 0.0 - return value - def _convert_callback_formats(callback): callback = getattr(callback, 'im_func', callback) if callback not in _converted_callback_formats: Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/generation.py Thu Dec 9 16:33:59 2010 @@ -2,7 +2,7 @@ from pypy.rpython.memory.gc.semispace import SemiSpaceGC from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL, GCFLAG_FORWARDED from pypy.rpython.memory.gc.semispace import GC_HASH_TAKEN_ADDR -from pypy.rpython.memory.gc.base import read_from_env +from pypy.rpython.memory.gc import env from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object @@ -93,7 +93,7 @@ if self.auto_nursery_size: newsize = nursery_size_from_env() if newsize <= 0: - newsize = estimate_best_nursery_size() + newsize = env.estimate_best_nursery_size() if newsize > 0: self.set_nursery_size(newsize) @@ -633,139 +633,5 @@ # ____________________________________________________________ -import os - def nursery_size_from_env(): - return read_from_env('PYPY_GENERATIONGC_NURSERY') - -def best_nursery_size_for_L2cache(L2cache): - # Heuristically, the best nursery size to choose is about half - # of the L2 cache. XXX benchmark some more. - return L2cache // 2 - - -if sys.platform == 'linux2': - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. - """ - debug_start("gc-L2cache") - L2cache = sys.maxint - try: - fd = os.open('/proc/cpuinfo', os.O_RDONLY, 0644) - try: - data = [] - while True: - buf = os.read(fd, 4096) - if not buf: - break - data.append(buf) - finally: - os.close(fd) - except OSError: - pass - else: - data = ''.join(data) - linepos = 0 - while True: - start = findend(data, '\ncache size', linepos) - if start < 0: - break # done - linepos = findend(data, '\n', start) - if linepos < 0: - break # no end-of-line?? - # *** data[start:linepos] == " : 2048 KB\n" - start = skipspace(data, start) - if data[start] != ':': - continue - # *** data[start:linepos] == ": 2048 KB\n" - start = skipspace(data, start + 1) - # *** data[start:linepos] == "2048 KB\n" - end = start - while '0' <= data[end] <= '9': - end += 1 - # *** data[start:end] == "2048" - if start == end: - continue - number = int(data[start:end]) - # *** data[end:linepos] == " KB\n" - end = skipspace(data, end) - if data[end] not in ('K', 'k'): # assume kilobytes for now - continue - number = number * 1024 - # for now we look for the smallest of the L2 caches of the CPUs - if number < L2cache: - L2cache = number - - debug_print("L2cache =", L2cache) - debug_stop("gc-L2cache") - - if L2cache < sys.maxint: - return best_nursery_size_for_L2cache(L2cache) - else: - # Print a top-level warning even in non-debug builds - llop.debug_print(lltype.Void, - "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") - return -1 - - def findend(data, pattern, pos): - pos = data.find(pattern, pos) - if pos < 0: - return -1 - return pos + len(pattern) - - def skipspace(data, pos): - while data[pos] in (' ', '\t'): - pos += 1 - return pos - -elif sys.platform == 'darwin': - from pypy.rpython.lltypesystem import rffi - - sysctlbyname = rffi.llexternal('sysctlbyname', - [rffi.CCHARP, rffi.VOIDP, rffi.SIZE_TP, - rffi.VOIDP, rffi.SIZE_T], - rffi.INT, - sandboxsafe=True) - - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. - """ - debug_start("gc-L2cache") - L2cache = 0 - l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') - try: - len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') - try: - size = rffi.sizeof(rffi.LONGLONG) - l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) - len_p[0] = rffi.cast(rffi.SIZE_T, size) - # XXX a hack for llhelper not being robust-enough - result = sysctlbyname("hw.l2cachesize", - rffi.cast(rffi.VOIDP, l2cache_p), - len_p, - lltype.nullptr(rffi.VOIDP.TO), - rffi.cast(rffi.SIZE_T, 0)) - if (rffi.cast(lltype.Signed, result) == 0 and - rffi.cast(lltype.Signed, len_p[0]) == size): - L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) - if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: - L2cache = 0 # overflow! - finally: - lltype.free(len_p, flavor='raw') - finally: - lltype.free(l2cache_p, flavor='raw') - debug_print("L2cache =", L2cache) - debug_stop("gc-L2cache") - if L2cache > 0: - return best_nursery_size_for_L2cache(L2cache) - else: - # Print a top-level warning even in non-debug builds - llop.debug_print(lltype.Void, - "Warning: cannot find your CPU L2 cache size with sysctl()") - return -1 - -else: - def estimate_best_nursery_size(): - return -1 # XXX implement me for other platforms + return env.read_from_env('PYPY_GENERATIONGC_NURSERY') Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/inspector.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/inspector.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/inspector.py Thu Dec 9 16:33:59 2010 @@ -101,21 +101,24 @@ AddressStack = get_address_stack() -class HeapDumper: +class HeapDumper(object): _alloc_flavor_ = "raw" BUFSIZE = 8192 # words def __init__(self, gc, fd): self.gc = gc + self.gcflag = gc.gcflag_extra self.fd = rffi.cast(rffi.INT, fd) self.writebuffer = lltype.malloc(rffi.LONGP.TO, self.BUFSIZE, flavor='raw') self.buf_count = 0 - self.seen = AddressDict() + if self.gcflag == 0: + self.seen = AddressDict() self.pending = AddressStack() def delete(self): - self.seen.delete() + if self.gcflag == 0: + self.seen.delete() self.pending.delete() lltype.free(self.writebuffer, flavor='raw') free_non_gc_object(self) @@ -140,6 +143,8 @@ self.flush() write._always_inline_ = True + # ---------- + def write_marker(self): self.write(0) self.write(0) @@ -161,9 +166,15 @@ self.add(obj) def add(self, obj): - if not self.seen.contains(obj): - self.seen.setitem(obj, obj) - self.pending.append(obj) + if self.gcflag == 0: + if not self.seen.contains(obj): + self.seen.setitem(obj, obj) + self.pending.append(obj) + else: + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) == 0: + hdr.tid |= self.gcflag + self.pending.append(obj) def add_roots(self): self.gc.enumerate_all_roots(_hd_add_root, self) @@ -177,14 +188,50 @@ while pending.non_empty(): self.writeobj(pending.pop()) + # ---------- + # A simplified copy of the above, to make sure we walk again all the + # objects to clear the 'gcflag'. + + def unwriteobj(self, obj): + gc = self.gc + gc.trace(obj, self._unwriteref, None) + + def _unwriteref(self, pointer, _): + obj = pointer.address[0] + self.unadd(obj) + + def unadd(self, obj): + assert self.gcflag != 0 + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) != 0: + hdr.tid &= ~self.gcflag + self.pending.append(obj) + + def clear_gcflag_again(self): + self.gc.enumerate_all_roots(_hd_unadd_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.unwalk(pendingroots) + pendingroots.delete() + + def unwalk(self, pending): + while pending.non_empty(): + self.unwriteobj(pending.pop()) + def _hd_add_root(obj, heap_dumper): heap_dumper.add(obj) +def _hd_unadd_root(obj, heap_dumper): + heap_dumper.unadd(obj) + def dump_rpy_heap(gc, fd): heapdumper = HeapDumper(gc, fd) heapdumper.add_roots() heapdumper.walk(heapdumper.pending) heapdumper.flush() + if heapdumper.gcflag != 0: + heapdumper.clear_gcflag_again() + heapdumper.unwalk(heapdumper.pending) heapdumper.delete() return True Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/markcompact.py Thu Dec 9 16:33:59 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup -from pypy.rpython.memory.gc.base import MovingGCBase, read_from_env +from pypy.rpython.memory.gc.base import MovingGCBase +from pypy.rpython.memory.gc import env from pypy.rlib.debug import ll_assert, have_debug_prints from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.support import get_address_stack, get_address_deque @@ -110,10 +111,10 @@ return next def setup(self): - envsize = read_from_env('PYPY_MARKCOMPACTGC_MAX') + envsize = env.read_from_env('PYPY_MARKCOMPACTGC_MAX') if envsize >= 4096: self.space_size = envsize & ~4095 - mincollect = read_from_env('PYPY_MARKCOMPACTGC_MIN') + mincollect = env.read_from_env('PYPY_MARKCOMPACTGC_MIN') if mincollect >= 4096: self.min_next_collect_after = mincollect Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/minimark.py Thu Dec 9 16:33:59 2010 @@ -3,7 +3,8 @@ Environment variables can be used to fine-tune the following parameters: PYPY_GC_NURSERY The nursery size. Defaults to half the size of - the L2 cache. Try values like '1.2MB'. + the L2 cache. Try values like '1.2MB'. Small values + (like 1 or 1KB) are useful for debugging. PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', which means trigger a major collection when the @@ -12,7 +13,7 @@ collection. PYPY_GC_GROWTH Major collection threshold's max growth rate. - Default is '1.3'. Useful to collect more often + Default is '1.4'. Useful to collect more often than normally on sudden memory growth, e.g. when there is a temporary peak in memory usage. @@ -22,10 +23,21 @@ crash the program with a fatal error. Try values like '1.6GB'. + PYPY_GC_MAX_DELTA The major collection threshold will never be set + to more than PYPY_GC_MAX_DELTA the amount really + used after a collection. Defaults to 1/8th of the + total RAM size (which is constrained to be at most + 2/3/4GB on 32-bit systems). Try values like '200MB'. + PYPY_GC_MIN Don't collect while the memory size is below this limit. Useful to avoid spending all the time in the GC in very small programs. Defaults to 8 times the nursery. + + PYPY_GC_DEBUG Enable extra checks around collections that are + too slow for normal use. Values are 0 (off), + 1 (on major collections) or 2 (also on minor + collections). """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -36,7 +48,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage from pypy.rpython.memory.gc.base import GCBase, MovingGCBase -from pypy.rpython.memory.gc import minimarkpage, base, generation +from pypy.rpython.memory.gc import minimarkpage, env from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint from pypy.rlib.rarithmetic import LONG_BIT_SHIFT from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop @@ -96,6 +108,7 @@ needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False malloc_zero_filled = True # xxx experiment with False + gcflag_extra = GCFLAG_FINALIZATION_ORDERING # All objects start with a HDR, i.e. with a field 'tid' which contains # a word. This word is divided in two halves: the lower half contains @@ -154,7 +167,7 @@ # grow at most by the following factor from one collection to the # next. Used e.g. when there is a sudden, temporary peak in memory # usage; this avoids that the upper bound grows too fast. - "growth_rate_max": 1.3, + "growth_rate_max": 1.4, # The number of array indices that are mapped to a single bit in # write_barrier_from_array(). Must be a power of two. The default @@ -198,6 +211,7 @@ self.min_heap_size = 0.0 self.max_heap_size = 0.0 self.max_heap_size_already_raised = False + self.max_delta = float(r_uint(-1)) # self.card_page_indices = card_page_indices if self.card_page_indices > 0: @@ -215,7 +229,8 @@ self.nursery = NULL self.nursery_free = NULL self.nursery_top = NULL - self.debug_always_do_minor_collect = False + self.debug_tiny_nursery = -1 + self.debug_rotating_nurseries = None # # The ArenaCollection() handles the nonmovable objects allocation. if ArenaCollectionClass is None: @@ -282,53 +297,71 @@ # # From there on, the GC is fully initialized and the code # below can use it - newsize = base.read_from_env('PYPY_GC_NURSERY') - # PYPY_GC_NURSERY=1 forces a minor collect for every malloc. - # Useful to debug external factors, like trackgcroot or the - # handling of the write barrier. - self.debug_always_do_minor_collect = newsize == 1 + newsize = env.read_from_env('PYPY_GC_NURSERY') + # PYPY_GC_NURSERY=smallvalue means that minor collects occur + # very frequently; the extreme case is PYPY_GC_NURSERY=1, which + # forces a minor collect for every malloc. Useful to debug + # external factors, like trackgcroot or the handling of the write + # barrier. Implemented by still using 'minsize' for the nursery + # size (needed to handle e.g. mallocs of 8249 words) but hacking + # at the current nursery position in collect_and_reserve(). if newsize <= 0: - newsize = generation.estimate_best_nursery_size() + newsize = env.estimate_best_nursery_size() if newsize <= 0: newsize = defaultsize - newsize = max(newsize, minsize) + if newsize < minsize: + self.debug_tiny_nursery = newsize & ~(WORD-1) + newsize = minsize # - major_coll = base.read_float_from_env('PYPY_GC_MAJOR_COLLECT') + major_coll = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT') if major_coll > 1.0: self.major_collection_threshold = major_coll # - growth = base.read_float_from_env('PYPY_GC_GROWTH') + growth = env.read_float_from_env('PYPY_GC_GROWTH') if growth > 1.0: self.growth_rate_max = growth # - min_heap_size = base.read_uint_from_env('PYPY_GC_MIN') + min_heap_size = env.read_uint_from_env('PYPY_GC_MIN') if min_heap_size > 0: self.min_heap_size = float(min_heap_size) else: # defaults to 8 times the nursery self.min_heap_size = newsize * 8 # - max_heap_size = base.read_uint_from_env('PYPY_GC_MAX') + max_heap_size = env.read_uint_from_env('PYPY_GC_MAX') if max_heap_size > 0: self.max_heap_size = float(max_heap_size) # + max_delta = env.read_uint_from_env('PYPY_GC_MAX_DELTA') + if max_delta > 0: + self.max_delta = float(max_delta) + else: + self.max_delta = 0.125 * env.get_total_memory() + # self.minor_collection() # to empty the nursery llarena.arena_free(self.nursery) self.nursery_size = newsize self.allocate_nursery() - def allocate_nursery(self): - debug_start("gc-set-nursery-size") - debug_print("nursery size:", self.nursery_size) + def _nursery_memory_size(self): + extra = self.nonlarge_gcptrs_max + 1 + return self.nursery_size + extra + + def _alloc_nursery(self): # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize_clear(). The few extra pages are never used # anyway so it doesn't even count. - extra = self.nonlarge_gcptrs_max + 1 - self.nursery = llarena.arena_malloc(self.nursery_size + extra, 2) - if not self.nursery: + nursery = llarena.arena_malloc(self._nursery_memory_size(), 2) + if not nursery: raise MemoryError("cannot allocate nursery") + return nursery + + def allocate_nursery(self): + debug_start("gc-set-nursery-size") + debug_print("nursery size:", self.nursery_size) + self.nursery = self._alloc_nursery() # the current position in the nursery: self.nursery_free = self.nursery # the end of the nursery: @@ -362,6 +395,39 @@ return bounded + def post_setup(self): + # set up extra stuff for PYPY_GC_DEBUG. + MovingGCBase.post_setup(self) + if self.DEBUG and llarena.has_protect: + # gc debug mode: allocate 23 nurseries instead of just 1, + # and use them alternatively, while mprotect()ing the unused + # ones to detect invalid access. + debug_start("gc-debug") + self.debug_rotating_nurseries = [] + for i in range(22): + nurs = self._alloc_nursery() + llarena.arena_protect(nurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(nurs) + debug_print("allocated", len(self.debug_rotating_nurseries), + "extra nurseries") + debug_stop("gc-debug") + + def debug_rotate_nursery(self): + if self.debug_rotating_nurseries is not None: + debug_start("gc-debug") + oldnurs = self.nursery + llarena.arena_protect(oldnurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(oldnurs) + # + newnurs = self.debug_rotating_nurseries.pop(0) + llarena.arena_protect(newnurs, self._nursery_memory_size(), False) + self.nursery = newnurs + self.nursery_top = self.nursery + self.nursery_size + debug_print("switching to nursery", self.nursery, + "size", self.nursery_size) + debug_stop("gc-debug") + + def malloc_fixedsize_clear(self, typeid, size, can_collect=True, needs_finalizer=False, contains_weakptr=False): ll_assert(can_collect, "!can_collect") @@ -499,8 +565,9 @@ self.nursery_free = result + totalsize ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow") # - if self.debug_always_do_minor_collect: - self.nursery_free = self.nursery_top + if self.debug_tiny_nursery >= 0: # for debugging + if self.nursery_top - self.nursery_free > self.debug_tiny_nursery: + self.nursery_free = self.nursery_top - self.debug_tiny_nursery # return result collect_and_reserve._dont_inline_ = True @@ -638,8 +705,13 @@ # means recording that they have a smaller size, so that when # moved out of the nursery, they will consume less memory. # In particular, an array with GCFLAG_HAS_CARDS is never resized. + # Also, a nursery object with GCFLAG_HAS_SHADOW is not resized + # either, as this would potentially loose part of the memory in + # the already-allocated shadow. if not self.is_in_nursery(obj): return False + if self.header(obj).tid & GCFLAG_HAS_SHADOW: + return False # size_gc_header = self.gcheaderbuilder.size_gc_header typeid = self.get_type_id(obj) @@ -822,7 +894,7 @@ def remember_young_pointer(addr_struct, newvalue): # 'addr_struct' is the address of the object in which we write. # 'newvalue' is the address that we are going to write in there. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") # @@ -859,7 +931,7 @@ # 'addr_array' is the address of the object in which we write, # which must have an array part; 'index' is the index of the # item that is (or contains) the pointer that we write. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_array), "nursery array with GCFLAG_NO_YOUNG_PTRS") objhdr = self.header(addr_array) @@ -981,13 +1053,14 @@ # All live nursery objects are out, and the rest dies. Fill # the whole nursery with zero and reset the current nursery pointer. llarena.arena_reset(self.nursery, self.nursery_size, 2) + self.debug_rotate_nursery() self.nursery_free = self.nursery # debug_print("minor collect, total memory used:", self.get_total_memory_used()) + if self.DEBUG >= 2: + self.debug_check_consistency() # expensive! debug_stop("gc-minor") - if 0: # not we_are_translated(): - self.debug_check_consistency() # xxx expensive! def collect_roots_in_nursery(self): @@ -1245,9 +1318,12 @@ # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than + # we currently have -- but no more than 'max_delta' more than # we currently have. + total_memory_used = float(self.get_total_memory_used()) bounded = self.set_major_threshold_from( - self.get_total_memory_used() * self.major_collection_threshold, + min(total_memory_used * self.major_collection_threshold, + total_memory_used + self.max_delta), reserving_size) # # Max heap size: gives an upper bound on the threshold. If we @@ -1406,12 +1482,21 @@ size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + size) - # initialize to an invalid tid *without* GCFLAG_VISITED, - # so that if the object dies before the next minor - # collection, the shadow will stay around but be collected - # by the next major collection. + # Initialize the shadow enough to be considered a + # valid gc object. If the original object stays + # alive at the next minor collection, it will anyway + # be copied over the shadow and overwrite the + # following fields. But if the object dies, then + # the shadow will stay around and only be freed at + # the next major collection, at which point we want + # it to look valid (but ready to be freed). shadow = shadowhdr + size_gc_header - self.header(shadow).tid = 0 + self.header(shadow).tid = self.header(obj).tid + typeid = self.get_type_id(obj) + if self.is_varsize(typeid): + lenofs = self.varsize_offset_to_length(typeid) + (shadow + lenofs).signed[0] = (obj + lenofs).signed[0] + # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.young_objects_shadows.setitem(obj, shadow) # Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/semispace.py Thu Dec 9 16:33:59 2010 @@ -42,6 +42,7 @@ inline_simple_malloc_varsize = True malloc_zero_filled = True first_unused_gcflag = first_gcflag << 5 + gcflag_extra = GCFLAG_FINALIZATION_ORDERING HDR = lltype.Struct('header', ('tid', lltype.Signed)) # XXX or rffi.INT? typeid_is_in_field = 'tid' Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gc/test/test_direct.py Thu Dec 9 16:33:59 2010 @@ -60,7 +60,7 @@ pass -class DirectGCTest(object): +class BaseDirectGCTest(object): GC_PARAMS = {} def setup_method(self, meth): @@ -106,6 +106,9 @@ addr = self.gc.malloc(self.get_type_id(TYPE), n, zero=True) return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) + +class DirectGCTest(BaseDirectGCTest): + def test_simple(self): p = self.malloc(S) p.x = 5 @@ -339,6 +342,15 @@ self.gc.collect() assert hash == self.gc.identityhash(self.stackroots[-1]) self.stackroots.pop() + # (7) the same, but the objects are dying young + for i in range(10): + self.gc.collect() + p = self.malloc(VAR, i) + self.stackroots.append(p) + hash1 = self.gc.identityhash(p) + hash2 = self.gc.identityhash(p) + assert hash1 == hash2 + self.stackroots.pop() def test_memory_alignment(self): A1 = lltype.GcArray(lltype.Char) Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/asmgcroot.py Thu Dec 9 16:33:59 2010 @@ -139,12 +139,13 @@ self._shape_decompressor = ShapeDecompressor() if hasattr(gctransformer.translator, '_jit2gc'): jit2gc = gctransformer.translator._jit2gc - self._extra_gcmapstart = jit2gc['gcmapstart'] - self._extra_gcmapend = jit2gc['gcmapend'] + self._extra_gcmapstart = jit2gc['gcmapstart'] + self._extra_gcmapend = jit2gc['gcmapend'] + self._extra_mark_sorted = jit2gc['gcmarksorted'] else: - returns_null = lambda: llmemory.NULL - self._extra_gcmapstart = returns_null - self._extra_gcmapend = returns_null + self._extra_gcmapstart = lambda: llmemory.NULL + self._extra_gcmapend = lambda: llmemory.NULL + self._extra_mark_sorted = lambda: True def need_thread_support(self, gctransformer, getfn): # Threads supported "out of the box" by the rest of the code. @@ -295,14 +296,16 @@ # we have a non-empty JIT-produced table to look in item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) if item: - self._shape_decompressor.setaddr(item.address[1]) + self._shape_decompressor.setaddr(item) return # maybe the JIT-produced table is not sorted? - sort_gcmap(gcmapstart2, gcmapend2) - item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) - if item: - self._shape_decompressor.setaddr(item.address[1]) - return + was_already_sorted = self._extra_mark_sorted() + if not was_already_sorted: + sort_gcmap(gcmapstart2, gcmapend2) + item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) + if item: + self._shape_decompressor.setaddr(item) + return # the item may have been not found because the main array was # not sorted. Sort it and try again. win32_follow_gcmap_jmp(gcmapstart, gcmapend) @@ -357,7 +360,8 @@ The interval from the start address (included) to the end address (excluded) is assumed to be a sorted arrays of pairs (addr1, addr2). This searches for the item with a given addr1 and returns its - address. + address. If not found exactly, it tries to return the address + of the item left of addr1 (i.e. such that result.address[0] < addr1). """ count = (end - start) // arrayitemsize while count > 1: @@ -386,7 +390,7 @@ # (item.signed[1] is an address in this case, not a signed at all!) item = binary_search(gcmapstart, gcmapend, retaddr) if item.address[0] == retaddr: - return item # found + return item.address[1] # found else: return llmemory.NULL # failed Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/boehm.py Thu Dec 9 16:33:59 2010 @@ -120,10 +120,11 @@ fptr = self.annotate_finalizer(d['ll_finalizer'], [llmemory.Address], lltype.Void) elif destrptr: EXC_INSTANCE_TYPE = self.translator.rtyper.exceptiondata.lltype_of_exception_value + typename = TYPE.__name__ def ll_finalizer(addr): exc_instance = llop.gc_fetch_exception(EXC_INSTANCE_TYPE) v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) llop.gc_restore_exception(lltype.Void, exc_instance) fptr = self.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) else: Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/framework.py Thu Dec 9 16:33:59 2010 @@ -189,6 +189,7 @@ # run-time initialization code root_walker.setup_root_walker() gcdata.gc.setup() + gcdata.gc.post_setup() def frameworkgc__teardown(): # run-time teardown code for tests! @@ -1204,9 +1205,10 @@ assert not type_contains_pyobjs(TYPE), "not implemented" if destrptr: + typename = TYPE.__name__ def ll_finalizer(addr): v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) fptr = self.transformer.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/refcounting.py Thu Dec 9 16:33:59 2010 @@ -227,7 +227,7 @@ # refcount is at zero, temporarily bump it to 1: gcheader.refcount = 1 destr_v = cast_pointer(DESTR_ARG, v) - ll_call_destructor(destrptr, destr_v) + ll_call_destructor(destrptr, destr_v, %r) refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: @@ -239,7 +239,7 @@ pop_alive(exc_instance) # XXX layering of exceptiontransform versus gcpolicy -""" % (body, TYPE._gckind) +""" % (TYPE.__name__, body, TYPE._gckind) else: call_del = None body = '\n'.join(_static_deallocator_body_for_type('v', TYPE)) Modified: pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/memory/gctransform/support.py Thu Dec 9 16:33:59 2010 @@ -98,11 +98,15 @@ hop.exception_cannot_occur() return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const) -def ll_call_destructor(destrptr, destr_v): +def ll_call_destructor(destrptr, destr_v, typename): try: destrptr(destr_v) - except: + except Exception, e: try: - os.write(2, "a destructor raised an exception, ignoring it\n") + os.write(2, "a destructor of type ") + os.write(2, typename) + os.write(2, " raised an exception ") + os.write(2, str(e)) + os.write(2, " ignoring it\n") except: pass Modified: pypy/branch/out-of-line-guards/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/module/ll_os.py Thu Dec 9 16:33:59 2010 @@ -1386,6 +1386,25 @@ return extdef([], (int, int), "ll_os.ll_os_openpty", llimpl=openpty_llimpl) + @registering_if(os, 'forkpty') + def register_os_forkpty(self): + os_forkpty = self.llexternal( + 'forkpty', + [rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP], + rffi.PID_T, + compilation_info=ExternalCompilationInfo(libraries=['util'])) + def forkpty_llimpl(): + master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + childpid = os_forkpty(master_p, None, None, None) + master_fd = master_p[0] + lltype.free(master_p, flavor='raw') + if childpid == -1: + raise OSError(rposix.get_errno(), "os_forkpty failed") + return (rffi.cast(lltype.Signed, childpid), + rffi.cast(lltype.Signed, master_fd)) + + return extdef([], (int, int), "ll_os.ll_os_forkpty", + llimpl=forkpty_llimpl) @registering(os._exit) def register_os__exit(self): Modified: pypy/branch/out-of-line-guards/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/module/ll_time.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/module/ll_time.py Thu Dec 9 16:33:59 2010 @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if sys.platform == 'freebsd7': +if "freebsd" in sys.platform: libraries = ['compat'] else: libraries = [] Modified: pypy/branch/out-of-line-guards/pypy/rpython/ootypesystem/rpbc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/ootypesystem/rpbc.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/ootypesystem/rpbc.py Thu Dec 9 16:33:59 2010 @@ -49,7 +49,7 @@ return hop.genop('runtimenew', [v_class], resulttype=resulttype) def getlowleveltype(self): - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) # if any of the classdefs get the lowleveltype ootype.Class, # we can only pick ootype.Class for us too. Otherwise META. for classdesc in classdescs: @@ -70,7 +70,7 @@ class MethodImplementations(object): def __init__(self, rtyper, methdescs): - samplemdesc = methdescs.iterkeys().next() + samplemdesc = iter(methdescs).next() concretetable, uniquerows = get_concrete_calltable(rtyper, samplemdesc.funcdesc.getcallfamily()) self.row_mapping = {} @@ -117,7 +117,7 @@ concretetable = None # set by _setup_repr_final def _setup_repr_final(self): - sampledesc = self.s_pbc.descriptions.iterkeys().next() + sampledesc = self.s_pbc.any_description() self.concretetable, _ = get_concrete_calltable(self.rtyper, sampledesc.funcdesc.getcallfamily()) Modified: pypy/branch/out-of-line-guards/pypy/rpython/rlist.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/rlist.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/rlist.py Thu Dec 9 16:33:59 2010 @@ -9,7 +9,7 @@ from pypy.rpython import robject from pypy.rlib.objectmodel import malloc_zero_filled from pypy.rlib.debug import ll_assert -from pypy.rlib.rarithmetic import ovfcheck, widen +from pypy.rlib.rarithmetic import ovfcheck, widen, r_uint, intmask from pypy.rpython.annlowlevel import ADTInterface from pypy.rlib import rgc @@ -241,17 +241,22 @@ class __extend__(pairtype(AbstractBaseListRepr, IntegerRepr)): def rtype_getitem((r_lst, r_int), hop, checkidx=False): + v_lst, v_index = hop.inputargs(r_lst, Signed) if checkidx: - spec = dum_checkidx + hop.exception_is_here() else: - spec = dum_nocheck - v_func = hop.inputconst(Void, spec) - v_lst, v_index = hop.inputargs(r_lst, Signed) + hop.exception_cannot_occur() if hop.args_s[0].listdef.listitem.mutated or checkidx: if hop.args_s[1].nonneg: llfn = ll_getitem_nonneg else: llfn = ll_getitem + if checkidx: + spec = dum_checkidx + else: + spec = dum_nocheck + c_func_marker = hop.inputconst(Void, spec) + v_res = hop.gendirectcall(llfn, c_func_marker, v_lst, v_index) else: # this is the 'foldable' version, which is not used when # we check for IndexError @@ -259,11 +264,7 @@ llfn = ll_getitem_foldable_nonneg else: llfn = ll_getitem_foldable - if checkidx: - hop.exception_is_here() - else: - hop.exception_cannot_occur() - v_res = hop.gendirectcall(llfn, v_func, v_lst, v_index) + v_res = hop.gendirectcall(llfn, v_lst, v_index) return r_lst.recast(hop.llops, v_res) rtype_getitem_key = rtype_getitem @@ -538,12 +539,14 @@ dest.ll_setitem_fast(dest_start + i, item) i += 1 ll_arraycopy._annenforceargs_ = [None, None, int, int, int] +# no oopspec -- the function is inlined by the JIT def ll_copy(RESLIST, l): length = l.ll_length() new_lst = RESLIST.ll_newlist(length) ll_arraycopy(l, new_lst, 0, 0, length) return new_lst +# no oopspec -- the function is inlined by the JIT def ll_len(l): return l.ll_length() @@ -551,6 +554,7 @@ def ll_list_is_true(l): # check if a list is True, allowing for None return bool(l) and l.ll_length() != 0 +# no oopspec -- the function is inlined by the JIT def ll_len_foldable(l): return l.ll_length() @@ -558,6 +562,7 @@ def ll_list_is_true_foldable(l): return bool(l) and ll_len_foldable(l) != 0 +# no oopspec -- the function is inlined by the JIT def ll_append(l, newitem): length = l.ll_length() @@ -588,6 +593,7 @@ ll_arraycopy(l1, l, 0, 0, len1) ll_arraycopy(l2, l, 0, len1, len2) return l +# no oopspec -- the function is inlined by the JIT def ll_insert_nonneg(l, index, newitem): length = l.ll_length() @@ -674,60 +680,72 @@ l.ll_setitem_fast(length_1_i, tmp) i += 1 length_1_i -= 1 +ll_reverse.oopspec = 'list.reverse(l)' def ll_getitem_nonneg(func, l, index): ll_assert(index >= 0, "unexpectedly negative list getitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError - else: - ll_assert(index < l.ll_length(), "list getitem index out of bound") return l.ll_getitem_fast(index) -ll_getitem_nonneg.oopspec = 'list.getitem(l, index)' +ll_getitem_nonneg._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_getitem(func, l, index): - length = l.ll_length() - if index < 0: - index += length if func is dum_checkidx: - if index < 0 or index >= length: - raise IndexError + length = l.ll_length() # common case: 0 <= index < length + if r_uint(index) >= r_uint(length): + # Failed, so either (-length <= index < 0), or we have to raise + # IndexError. First add 'length' to get the final index, then + # check that we now have (0 <= index < length). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(index >= 0, "negative list getitem index out of bound") - ll_assert(index < length, "list getitem index out of bound") + # We don't want checking, but still want to support index < 0. + # Only call ll_length() if needed. + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list getitem index out of bound") return l.ll_getitem_fast(index) -ll_getitem.oopspec = 'list.getitem(l, index)' +# no oopspec -- the function is inlined by the JIT -def ll_getitem_foldable_nonneg(func, l, index): - return ll_getitem_nonneg(func, l, index) +def ll_getitem_foldable_nonneg(l, index): + ll_assert(index >= 0, "unexpectedly negative list getitem index") + return l.ll_getitem_fast(index) ll_getitem_foldable_nonneg.oopspec = 'list.getitem_foldable(l, index)' -def ll_getitem_foldable(func, l, index): - return ll_getitem(func, l, index) -ll_getitem_foldable.oopspec = 'list.getitem_foldable(l, index)' +def ll_getitem_foldable(l, index): + if index < 0: + index += l.ll_length() + return ll_getitem_foldable_nonneg(l, index) +ll_getitem_foldable._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_setitem_nonneg(func, l, index, newitem): ll_assert(index >= 0, "unexpectedly negative list setitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError - else: - ll_assert(index < l.ll_length(), "list setitem index out of bound") l.ll_setitem_fast(index, newitem) -ll_setitem_nonneg.oopspec = 'list.setitem(l, index, newitem)' +ll_setitem_nonneg._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_setitem(func, l, index, newitem): - length = l.ll_length() - if index < 0: - index += length if func is dum_checkidx: - if index < 0 or index >= length: - raise IndexError + length = l.ll_length() + if r_uint(index) >= r_uint(length): # see comments in ll_getitem(). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(index >= 0, "negative list setitem index out of bound") - ll_assert(index < length, "list setitem index out of bound") + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list setitem index out of bound") l.ll_setitem_fast(index, newitem) -ll_setitem.oopspec = 'list.setitem(l, index, newitem)' +# no oopspec -- the function is inlined by the JIT def ll_delitem_nonneg(func, l, index): ll_assert(index >= 0, "unexpectedly negative list delitem index") @@ -751,19 +769,20 @@ l._ll_resize_le(newlength) ll_delitem_nonneg.oopspec = 'list.delitem(l, index)' -def ll_delitem(func, l, i): - length = l.ll_length() - if i < 0: - i += length +def ll_delitem(func, l, index): if func is dum_checkidx: - if i < 0 or i >= length: - raise IndexError + length = l.ll_length() + if r_uint(index) >= r_uint(length): # see comments in ll_getitem(). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(i >= 0, "negative list delitem index out of bound") - ll_assert(i < length, "list delitem index out of bound") - ll_delitem_nonneg(dum_nocheck, l, i) -ll_delitem.oopspec = 'list.delitem(l, i)' - + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list delitem index out of bound") + ll_delitem_nonneg(dum_nocheck, l, index) +# no oopspec -- the function is inlined by the JIT def ll_extend(l1, l2): len1 = l1.ll_length() @@ -799,6 +818,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_str_slice_startstop(lst, s, getstrlen, getstritem, start, stop): @@ -824,6 +844,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_str_slice_minusone(lst, s, getstrlen, getstritem): len1 = lst.ll_length() @@ -843,6 +864,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_char_count(lst, char, count): if count <= 0: @@ -859,6 +881,7 @@ while j < newlength: lst.ll_setitem_fast(j, char) j += 1 +# not inlined by the JIT -- contains a loop def ll_listslice_startonly(RESLIST, l1, start): len1 = l1.ll_length() @@ -869,6 +892,7 @@ ll_arraycopy(l1, l, start, 0, newlength) return l ll_listslice_startonly._annenforceargs_ = (None, None, int) +# no oopspec -- the function is inlined by the JIT def ll_listslice_startstop(RESLIST, l1, start, stop): length = l1.ll_length() @@ -881,6 +905,7 @@ l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, start, 0, newlength) return l +# no oopspec -- the function is inlined by the JIT def ll_listslice_minusone(RESLIST, l1): newlength = l1.ll_length() - 1 @@ -888,6 +913,7 @@ l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, 0, 0, newlength) return l +# no oopspec -- the function is inlined by the JIT def ll_listdelslice_startonly(l, start): ll_assert(start >= 0, "del l[start:] with unexpectedly negative start") @@ -958,6 +984,7 @@ return False j += 1 return True +# not inlined by the JIT -- contains a loop def ll_listcontains(lst, obj, eqfn): lng = lst.ll_length() @@ -971,6 +998,7 @@ return True j += 1 return False +# not inlined by the JIT -- contains a loop def ll_listindex(lst, obj, eqfn): lng = lst.ll_length() @@ -984,6 +1012,7 @@ return j j += 1 raise ValueError # can't say 'list.index(x): x not in list' +# not inlined by the JIT -- contains a loop def ll_listremove(lst, obj, eqfn): index = ll_listindex(lst, obj, eqfn) # raises ValueError if obj not in lst @@ -1030,3 +1059,4 @@ i += 1 j += length return res +# not inlined by the JIT -- contains a loop Modified: pypy/branch/out-of-line-guards/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/rpbc.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/rpbc.py Thu Dec 9 16:33:59 2010 @@ -15,11 +15,10 @@ from pypy.rpython import callparse - def small_cand(rtyper, s_pbc): if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets and \ hasattr(rtyper.type_system.rpbc, 'SmallFunctionSetPBCRepr'): - callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(rtyper, callfamily) if len(uniquerows) == 1 and (not s_pbc.subset_of or small_cand(rtyper, s_pbc.subset_of)): return True @@ -31,7 +30,7 @@ return none_frozen_pbc_repr kind = self.getKind() if issubclass(kind, description.FunctionDesc): - sample = self.descriptions.keys()[0] + sample = self.any_description() callfamily = sample.querycallfamily() if callfamily and callfamily.total_calltable_size > 0: if sample.overridden: @@ -181,7 +180,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None: # a single function self.lowleveltype = Void @@ -207,7 +206,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) ## def function_signatures(self): @@ -322,7 +321,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness @@ -368,7 +367,7 @@ return robject.pyobj_repr def getFrozenPBCRepr(rtyper, s_pbc): - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) assert len(descs) >= 1 if len(descs) == 1 and not s_pbc.can_be_None: return SingleFrozenPBCRepr(descs[0]) @@ -530,7 +529,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper - self.funcdesc = s_pbc.descriptions.keys()[0].funcdesc + self.funcdesc = s_pbc.any_description().funcdesc # a hack to force the underlying function to show up in call_families # (generally not needed, as normalizecalls() should ensure this, @@ -662,7 +661,7 @@ and the ClassRepr of the class which stores this attribute in its vtable. """ - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) access = classdescs[0].queryattrfamily(attrname) for classdesc in classdescs[1:]: access1 = classdesc.queryattrfamily(attrname) @@ -819,7 +818,7 @@ if s_pbc.isNone(): raise TyperError("unsupported: variable of type " "bound-method-object or None") - mdescs = s_pbc.descriptions.keys() + mdescs = list(s_pbc.descriptions) methodname = mdescs[0].name classdef = mdescs[0].selfclassdef flags = mdescs[0].flags Modified: pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/test/test_rclass.py Thu Dec 9 16:33:59 2010 @@ -910,7 +910,7 @@ assert destrptr is not None def test_del_inheritance(self): - import gc + from pypy.rlib import rgc class State: pass s = State() @@ -931,8 +931,7 @@ A() B() C() - gc.collect() - gc.collect() + rgc.collect() return s.a_dels * 10 + s.b_dels res = f() assert res == 42 @@ -1094,6 +1093,7 @@ assert meth.finalizer def test_del_inheritance(self): + from pypy.rlib import rgc class State: pass s = State() @@ -1114,6 +1114,7 @@ A() B() C() + rgc.collect() return s.a_dels * 10 + s.b_dels res = f() assert res == 42 Modified: pypy/branch/out-of-line-guards/pypy/rpython/test/test_rint.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/test/test_rint.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/test/test_rint.py Thu Dec 9 16:33:59 2010 @@ -4,14 +4,9 @@ from pypy.annotation import model as annmodel from pypy.rpython.test import snippet from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong -from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.rarithmetic import ovfcheck, r_int64 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -if r_longlong is not r_int: - int64 = r_longlong -else: - int64 = int - class TestSnippet(object): @@ -110,10 +105,10 @@ def f(i): return str(i) - res = self.interpret(f, [int64(0)]) + res = self.interpret(f, [r_int64(0)]) assert self.ll_to_string(res) == '0' - res = self.interpret(f, [int64(413974738222117)]) + res = self.interpret(f, [r_int64(413974738222117)]) assert self.ll_to_string(res) == '413974738222117' def test_unsigned(self): @@ -135,7 +130,7 @@ f._annspecialcase_ = "specialize:argtype(0)" def g(n): if n > 0: - return f(int64(0)) + return f(r_int64(0)) else: return f(0) res = self.interpret(g, [0]) @@ -147,7 +142,7 @@ def test_downcast_int(self): def f(i): return int(i) - res = self.interpret(f, [int64(0)]) + res = self.interpret(f, [r_int64(0)]) assert res == 0 def test_isinstance_vs_int_types(self): @@ -157,7 +152,7 @@ return [None] if isinstance(x, str): return x - if isinstance(x, int64): + if isinstance(x, r_int64): return int(x) return "XXX" wrap._annspecialcase_ = 'specialize:argtype(0)' @@ -165,7 +160,7 @@ space = FakeSpace() def wrap(x): return space.wrap(x) - res = self.interpret(wrap, [int64(0)]) + res = self.interpret(wrap, [r_int64(0)]) assert res == 0 def test_truediv(self): @@ -178,25 +173,25 @@ def test_float_conversion(self): def f(ii): return float(ii) - res = self.interpret(f, [int64(100000000)]) + res = self.interpret(f, [r_int64(100000000)]) assert type(res) is float assert res == 100000000. - res = self.interpret(f, [int64(1234567890123456789)]) + res = self.interpret(f, [r_int64(1234567890123456789)]) assert type(res) is float assert self.float_eq(res, 1.2345678901234568e+18) def test_float_conversion_implicit(self): def f(ii): return 1.0 + ii - res = self.interpret(f, [int64(100000000)]) + res = self.interpret(f, [r_int64(100000000)]) assert type(res) is float assert res == 100000001. - res = self.interpret(f, [int64(1234567890123456789)]) + res = self.interpret(f, [r_int64(1234567890123456789)]) assert type(res) is float assert self.float_eq(res, 1.2345678901234568e+18) def test_rarithmetic(self): - inttypes = [int, r_uint, int64, r_ulonglong] + inttypes = [int, r_uint, r_int64, r_ulonglong] for inttype in inttypes: c = inttype() def f(): @@ -231,16 +226,16 @@ res = self.interpret(f, [int(-1<<(r_int.BITS-1))]) assert res == 0 - res = self.interpret(f, [int64(-1)]) + res = self.interpret(f, [r_int64(-1)]) assert res == 1 - res = self.interpret(f, [int64(-1)<<(r_longlong.BITS-1)]) + res = self.interpret(f, [r_int64(-1)<<(r_longlong.BITS-1)]) assert res == 0 div_mod_iteration_count = 1000 def test_div_mod(self): import random - for inttype in (int, int64): + for inttype in (int, r_int64): def d(x, y): return x/y @@ -303,7 +298,7 @@ except ZeroDivisionError: return 84 - for inttype in (int, int64): + for inttype in (int, r_int64): args = [( 5, 2), (-5, 2), ( 5,-2), (-5,-2), ( 6, 2), (-6, 2), ( 6,-2), (-6,-2), Modified: pypy/branch/out-of-line-guards/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/test/test_rlist.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/test/test_rlist.py Thu Dec 9 16:33:59 2010 @@ -12,6 +12,7 @@ from pypy.rpython.rint import signed_repr from pypy.objspace.flow.model import Constant, Variable from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +from pypy.rlib.debug import ll_assert # undo the specialization parameter for n1 in 'get set del'.split(): @@ -1076,7 +1077,13 @@ res = self.interpret(f, [0]) assert res == 1 - py.test.raises(AssertionError, self.interpret, f, [1]) + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) + else: + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def f(x): l = [1] @@ -1121,12 +1128,13 @@ res = self.interpret(f, [0]) assert res == 1 - try: - self.interpret_raises(IndexError, f, [1]) - except (AssertionError,), e: - pass + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) else: - assert False + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def f(x): l = [1] @@ -1163,12 +1171,13 @@ res = self.interpret(f, [0]) assert res == 1 - try: - self.interpret_raises(IndexError, f, [1]) - except (AssertionError,), e: - pass + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) else: - assert False + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def test_charlist_extension_1(self): def f(n): @@ -1327,8 +1336,32 @@ res = self.interpret(f, [2]) assert res == True + def test_immutable_list_out_of_instance(self): + from pypy.translator.simplify import get_funcobj + for immutable_fields in (["a", "b"], ["a", "b", "y[*]"]): + class A(object): + _immutable_fields_ = immutable_fields + class B(A): + pass + def f(i): + b = B() + lst = [i] + lst[0] += 1 + b.y = lst + ll_assert(b.y is lst, "copying when reading out the attr?") + return b.y[0] + res = self.interpret(f, [10]) + assert res == 11 + t, rtyper, graph = self.gengraph(f, [int]) + block = graph.startblock + op = block.operations[-1] + assert op.opname == 'direct_call' + func = get_funcobj(op.args[0].value)._callable + assert ('foldable' in func.func_name) == \ + ("y[*]" in immutable_fields) class TestLLtype(BaseTestRlist, LLRtypeMixin): + type_system = 'lltype' rlist = ll_rlist def test_memoryerror(self): @@ -1420,14 +1453,16 @@ lst2 = [i] lst2.append(42) # mutated list return lst1[i] + lst2[i] - _, _, graph = self.gengraph(f, [int]) + from pypy.annotation import model as annmodel + _, _, graph = self.gengraph(f, [annmodel.SomeInteger(nonneg=True)]) block = graph.startblock lst1_getitem_op = block.operations[-3] # XXX graph fishing lst2_getitem_op = block.operations[-2] func1 = lst1_getitem_op.args[0].value._obj._callable func2 = lst2_getitem_op.args[0].value._obj._callable assert func1.oopspec == 'list.getitem_foldable(l, index)' - assert func2.oopspec == 'list.getitem(l, index)' + assert not hasattr(func2, 'oopspec') class TestOOtype(BaseTestRlist, OORtypeMixin): rlist = oo_rlist + type_system = 'ootype' Modified: pypy/branch/out-of-line-guards/pypy/tool/ansi_print.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/tool/ansi_print.py (original) +++ pypy/branch/out-of-line-guards/pypy/tool/ansi_print.py Thu Dec 9 16:33:59 2010 @@ -16,6 +16,7 @@ 'WARNING': ((31,), False), 'event': ((1,), True), 'ERROR': ((1, 31), False), + 'Error': ((1, 31), False), 'info': ((35,), False), 'stub': ((34,), False), } Modified: pypy/branch/out-of-line-guards/pypy/tool/error.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/tool/error.py (original) +++ pypy/branch/out-of-line-guards/pypy/tool/error.py Thu Dec 9 16:33:59 2010 @@ -120,7 +120,7 @@ msg.append(" (%s getting at the binding!)" % ( e.__class__.__name__,)) return - for desc in descs.keys(): + for desc in list(descs): func = desc.pyobj if func is None: r = repr(desc) Modified: pypy/branch/out-of-line-guards/pypy/tool/logparser.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/tool/logparser.py (original) +++ pypy/branch/out-of-line-guards/pypy/tool/logparser.py Thu Dec 9 16:33:59 2010 @@ -12,14 +12,6 @@ from pypy.tool import progressbar def parse_log_file(filename, verbose=True): - r_start = re.compile(r"\[([0-9a-fA-F]+)\] \{([\w-]+)$") - r_stop = re.compile(r"\[([0-9a-fA-F]+)\] ([\w-]+)\}$") - lasttime = 0 - log = DebugLog() - time_decrase = False - performance_log = True - nested = 0 - # f = open(filename, 'r') if f.read(2) == 'BZ': f.close() @@ -30,19 +22,33 @@ lines = f.readlines() f.close() # - if sys.stdout.isatty(): + return parse_log(lines, verbose=verbose) + +def parse_log(lines, verbose=False): + color = "(?:\x1b.*?m)?" + r_start = re.compile(color + r"\[([0-9a-fA-F]+)\] \{([\w-]+)" + color + "$") + r_stop = re.compile(color + r"\[([0-9a-fA-F]+)\] ([\w-]+)\}" + color + "$") + lasttime = 0 + log = DebugLog() + time_decrase = False + performance_log = True + nested = 0 + # + if verbose and sys.stdout.isatty(): progress = progressbar.ProgressBar(color='green') + counter = 0 + else: + progress = None single_percent = len(lines) / 100 if verbose: - vnext = single_percent + vnext = 0 else: - vnext = len(lines) - counter = 0 + vnext = -1 for i, line in enumerate(lines): if i == vnext: - counter += 1 - if sys.stdout.isatty(): + if progress is not None: progress.render(counter) + counter += 1 vnext += single_percent else: sys.stderr.write('%d%%..' % int(100.0*i/len(lines))) Modified: pypy/branch/out-of-line-guards/pypy/tool/release/force-builds.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/tool/release/force-builds.py (original) +++ pypy/branch/out-of-line-guards/pypy/tool/release/force-builds.py Thu Dec 9 16:33:59 2010 @@ -20,8 +20,8 @@ 'own-linux-x86-32', 'own-linux-x86-64', # 'own-macosx-x86-32', - 'pypy-c-app-level-linux-x86-32', - 'pypy-c-app-level-linux-x86-64', +# 'pypy-c-app-level-linux-x86-32', +# 'pypy-c-app-level-linux-x86-64', 'pypy-c-stackless-app-level-linux-x86-32', 'pypy-c-app-level-win-x86-32', 'pypy-c-jit-linux-x86-32', Modified: pypy/branch/out-of-line-guards/pypy/tool/release/make_release.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/tool/release/make_release.py (original) +++ pypy/branch/out-of-line-guards/pypy/tool/release/make_release.py Thu Dec 9 16:33:59 2010 @@ -30,7 +30,8 @@ else: xml = override_xml dom = minidom.parseString(xml) - refs = [node.getAttribute('href') for node in dom.getElementsByTagName('a')] + refs = [node.getAttribute('href') for node in dom.getElementsByTagName('a') + if 'pypy' in node.getAttribute('href')] # all refs are of form: pypy-{type}-{revision}-{platform}.tar.bz2 r = re.compile('pypy-c-([\w\d]+)-(\d+)-([\w\d]+).tar.bz2$') d = {} Modified: pypy/branch/out-of-line-guards/pypy/tool/release/package.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/tool/release/package.py (original) +++ pypy/branch/out-of-line-guards/pypy/tool/release/package.py Thu Dec 9 16:33:59 2010 @@ -1,8 +1,12 @@ #!/usr/bin/env python """ A sample script that packages PyPy, provided that it's already built. -Usage: +It uses 'pypy/translator/goal/pypy-c' and parts of the rest of the working +copy. Usage: -package.py pypydir [name-of-archive] [name-of-pypy-c] + package.py root-pypy-dir [name-of-archive] [name-of-pypy-c] + +Usually you would do: package.py ../../.. pypy-VER-PLATFORM. +The output is found in the directory /tmp/usession-YOURNAME/build/. """ import autopath @@ -32,7 +36,7 @@ class PyPyCNotFound(Exception): pass -def package(basedir, name='pypy-nightly', rename_pypy_c='pypy-c', +def package(basedir, name='pypy-nightly', rename_pypy_c='pypy', copy_to_dir = None, override_pypy_c = None): basedir = py.path.local(basedir) if sys.platform == 'win32': @@ -64,13 +68,17 @@ headers = includedir.listdir('*.h') + includedir.listdir('*.inl') for n in headers: shutil.copy(str(n), str(pypydir.join('include'))) + # + spdir = pypydir.ensure('site-packages', dir=True) + shutil.copy(str(basedir.join('site-packages', 'README')), str(spdir)) + # pypydir.ensure('bin', dir=True) archive_pypy_c = pypydir.join('bin', rename_pypy_c) shutil.copy(str(pypy_c), str(archive_pypy_c)) old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) # ignore errors + os.system("strip -x " + str(archive_pypy_c)) # ignore errors if USE_TARFILE_MODULE: import tarfile tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') Modified: pypy/branch/out-of-line-guards/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/tool/release/test/test_package.py (original) +++ pypy/branch/out-of-line-guards/pypy/tool/release/test/test_package.py Thu Dec 9 16:33:59 2010 @@ -18,7 +18,7 @@ prefix = builddir.join(test) cpyver = '%d.%d.%d' % CPYTHON_VERSION[:3] assert prefix.join('lib-python', cpyver, 'test').check() - assert prefix.join('bin', 'pypy-c').check() + assert prefix.join('bin', 'pypy').check() assert prefix.join('lib_pypy', 'syslog.py').check() assert not prefix.join('lib_pypy', 'py').check() assert not prefix.join('lib_pypy', 'ctypes_configure').check() Modified: pypy/branch/out-of-line-guards/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/c/funcgen.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/c/funcgen.py Thu Dec 9 16:33:59 2010 @@ -1,3 +1,4 @@ +import sys from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring from pypy.translator.c.support import cdecl from pypy.translator.c.support import llvalue_from_constant, gen_assignments @@ -757,6 +758,16 @@ format.append('%s') argv.append('(%s) ? "True" : "False"' % self.expr(arg)) continue + elif T == SignedLongLong: + if sys.platform == 'win32': + format.append('%I64d') + else: + format.append('%lld') + elif T == UnsignedLongLong: + if sys.platform == 'win32': + format.append('%I64u') + else: + format.append('%llu') else: raise Exception("don't know how to debug_print %r" % (T,)) argv.append(self.expr(arg)) @@ -765,17 +776,20 @@ "if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}" % (', '.join(argv), free_line)) + def _op_debug(self, opname, arg): + if isinstance(arg, Constant): + string_literal = c_string_constant(''.join(arg.value.chars)) + return "%s(%s);" % (opname, string_literal) + else: + x = "%s(RPyString_AsCharP(%s));\n" % (opname, self.expr(arg)) + x += "RPyString_FreeCache();" + return x + def OP_DEBUG_START(self, op): - arg = op.args[0] - assert isinstance(arg, Constant) - return "PYPY_DEBUG_START(%s);" % ( - c_string_constant(''.join(arg.value.chars)),) + return self._op_debug('PYPY_DEBUG_START', op.args[0]) def OP_DEBUG_STOP(self, op): - arg = op.args[0] - assert isinstance(arg, Constant) - return "PYPY_DEBUG_STOP(%s);" % ( - c_string_constant(''.join(arg.value.chars)),) + return self._op_debug('PYPY_DEBUG_STOP', op.args[0]) def OP_DEBUG_ASSERT(self, op): return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]), Modified: pypy/branch/out-of-line-guards/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/c/gcc/test/test_trackgcroot.py Thu Dec 9 16:33:59 2010 @@ -98,14 +98,13 @@ """ lines = source.splitlines(True) parts = list(DarwinAssemblerParser().find_functions(iter(lines))) - assert len(parts) == 7 + assert len(parts) == 6 assert parts[0] == (False, lines[:3]) assert parts[1] == (True, lines[3:7]) assert parts[2] == (True, lines[7:11]) - assert parts[3] == (True, lines[11:13]) - assert parts[4] == (False, lines[13:18]) - assert parts[5] == (True, lines[18:20]) - assert parts[6] == (False, lines[20:]) + assert parts[3] == (True, lines[11:18]) + assert parts[4] == (True, lines[18:20]) + assert parts[5] == (False, lines[20:]) def test_computegcmaptable(): tests = [] Modified: pypy/branch/out-of-line-guards/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/c/gcc/trackgcroot.py Thu Dec 9 16:33:59 2010 @@ -1106,7 +1106,7 @@ format = 'darwin64' function_names_prefix = '_' - LABEL = ElfFunctionGcRootTracker32.LABEL + LABEL = ElfFunctionGcRootTracker64.LABEL r_jmptable_item = re.compile(r"\t.(?:long|quad)\t"+LABEL+"(-\"?[A-Za-z0-9$]+\"?)?\s*$") r_functionstart = re.compile(r"_(\w+):\s*$") @@ -1405,6 +1405,7 @@ 'const_data' ] r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") + sections_doesnt_end_function = {'cstring': True, 'const': True} def find_functions(self, iterlines): functionlines = [] @@ -1412,20 +1413,20 @@ in_function = False for n, line in enumerate(iterlines): if self.r_textstart.match(line): - assert not in_text, "unexpected repeated .text start: %d" % n in_text = True elif self.r_sectionstart.match(line): - if in_function: + sectionname = self.r_sectionstart.match(line).group(1) + if (in_function and + sectionname not in self.sections_doesnt_end_function): yield in_function, functionlines functionlines = [] + in_function = False in_text = False - in_function = False elif in_text and self.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines functionlines = [] in_function = True functionlines.append(line) - if functionlines: yield in_function, functionlines @@ -1442,23 +1443,6 @@ format = "mingw32" FunctionGcRootTracker = Mingw32FunctionGcRootTracker - def find_functions(self, iterlines): - functionlines = [] - in_text = False - in_function = False - for n, line in enumerate(iterlines): - if self.r_textstart.match(line): - in_text = True - elif self.r_sectionstart.match(line): - in_text = False - elif in_text and self.FunctionGcRootTracker.r_functionstart.match(line): - yield in_function, functionlines - functionlines = [] - in_function = True - functionlines.append(line) - if functionlines: - yield in_function, functionlines - class MsvcAssemblerParser(AssemblerParser): format = "msvc" FunctionGcRootTracker = MsvcFunctionGcRootTracker Modified: pypy/branch/out-of-line-guards/pypy/translator/c/src/debug_alloc.h ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/c/src/debug_alloc.h (original) +++ pypy/branch/out-of-line-guards/pypy/translator/c/src/debug_alloc.h Thu Dec 9 16:33:59 2010 @@ -1,5 +1,5 @@ /**************************************************************/ - /*** tracking raw mallocs and frees for debugging ***/ +/*** tracking raw mallocs and frees for debugging ***/ #ifndef RPY_ASSERT @@ -62,8 +62,8 @@ count++; if (count > 0) { - fprintf(stderr, "debug_alloc.h: %ld mallocs left", count); char *env = getenv("PYPY_ALLOC"); + fprintf(stderr, "debug_alloc.h: %ld mallocs left", count); if (env && *env) { fprintf(stderr, " (most recent first):\n"); Modified: pypy/branch/out-of-line-guards/pypy/translator/c/src/debug_print.h ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/c/src/debug_print.h (original) +++ pypy/branch/out-of-line-guards/pypy/translator/c/src/debug_print.h Thu Dec 9 16:33:59 2010 @@ -72,7 +72,11 @@ { char *filename = getenv("PYPYLOG"); if (filename) +#ifndef MS_WINDOWS unsetenv("PYPYLOG"); /* don't pass it to subprocesses */ +#else + putenv("PYPYLOG="); /* don't pass it to subprocesses */ +#endif if (filename && filename[0]) { char *colon = strchr(filename, ':'); Modified: pypy/branch/out-of-line-guards/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/c/src/mem.h (original) +++ pypy/branch/out-of-line-guards/pypy/translator/c/src/mem.h Thu Dec 9 16:33:59 2010 @@ -233,4 +233,4 @@ #define OP_GC_GET_RPY_MEMORY_USAGE(x, r) r = -1 #define OP_GC_GET_RPY_TYPE_INDEX(x, r) r = -1 #define OP_GC_IS_RPY_INSTANCE(x, r) r = 0 -#define OP_GC_DUMP_RPY_HEAP(r) r = 0 +#define OP_GC_DUMP_RPY_HEAP(fd, r) r = 0 Modified: pypy/branch/out-of-line-guards/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/c/test/test_newgc.py Thu Dec 9 16:33:59 2010 @@ -1064,14 +1064,16 @@ def test_get_rpy_type_index(self): self.run("get_rpy_type_index") - filename_dump = str(udir.join('test_dump_rpy_heap')) + filename1_dump = str(udir.join('test_dump_rpy_heap.1')) + filename2_dump = str(udir.join('test_dump_rpy_heap.2')) def define_dump_rpy_heap(self): U = lltype.GcForwardReference() U.become(lltype.GcStruct('U', ('next', lltype.Ptr(U)), ('x', lltype.Signed))) S = lltype.GcStruct('S', ('u', lltype.Ptr(U))) A = lltype.GcArray(lltype.Ptr(S)) - filename = self.filename_dump + filename1 = self.filename1_dump + filename2 = self.filename2_dump def fn(): s = lltype.malloc(S) @@ -1081,20 +1083,31 @@ a = lltype.malloc(A, 1000) s2 = lltype.malloc(S) # - fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0666) - rgc.dump_rpy_heap(fd) + fd1 = os.open(filename1, os.O_WRONLY | os.O_CREAT, 0666) + fd2 = os.open(filename2, os.O_WRONLY | os.O_CREAT, 0666) + rgc.dump_rpy_heap(fd1) + rgc.dump_rpy_heap(fd2) # try twice in a row keepalive_until_here(s2) keepalive_until_here(s) keepalive_until_here(a) - os.close(fd) + os.close(fd1) + os.close(fd2) return 0 return fn def test_dump_rpy_heap(self): self.run("dump_rpy_heap") - assert os.path.exists(self.filename_dump) - assert os.path.getsize(self.filename_dump) > 64 + for fn in [self.filename1_dump, self.filename2_dump]: + assert os.path.exists(fn) + assert os.path.getsize(fn) > 64 + f = open(self.filename1_dump) + data1 = f.read() + f.close() + f = open(self.filename2_dump) + data2 = f.read() + f.close() + assert data1 == data2 filename_dump_typeids_z = str(udir.join('test_typeids_z')) def define_write_typeids_z(self): Modified: pypy/branch/out-of-line-guards/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/c/test/test_standalone.py Thu Dec 9 16:33:59 2010 @@ -272,7 +272,7 @@ x = "got:" debug_start ("mycat") if have_debug_prints(): x += "b" - debug_print ("foo", 2, "bar", 3) + debug_print ("foo", r_longlong(2), "bar", 3) debug_start ("cat2") if have_debug_prints(): x += "c" debug_print ("baz") @@ -403,6 +403,20 @@ assert not err assert path.check(file=0) + def test_debug_print_start_stop_nonconst(self): + def entry_point(argv): + debug_start(argv[1]) + debug_print(argv[2]) + debug_stop(argv[1]) + return 0 + t, cbuilder = self.compile(entry_point) + out, err = cbuilder.cmdexec("foo bar", err=True, env={'PYPYLOG': ':-'}) + lines = err.splitlines() + assert '{foo' in lines[0] + assert 'bar' == lines[1] + assert 'foo}' in lines[2] + + def test_fatal_error(self): def g(x): if x == 1: Modified: pypy/branch/out-of-line-guards/pypy/translator/driver.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/driver.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/driver.py Thu Dec 9 16:33:59 2010 @@ -11,7 +11,7 @@ from pypy.annotation import policy as annpolicy import optparse from pypy.tool.udir import udir -from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS +from pypy.tool.debug_print import debug_start, debug_print, debug_stop from pypy.rlib.entrypoint import secondary_entrypoints import py @@ -37,13 +37,6 @@ 'c': 'lltype', } -JIT_DEBUG = { - 'off' : DEBUG_OFF, - 'profile' : DEBUG_PROFILE, - 'steps' : DEBUG_STEPS, - 'detailed' : DEBUG_DETAILED, -} - def backend_to_typesystem(backend): return _BACKEND_TO_TYPESYSTEM.get(backend, 'ootype') @@ -283,6 +276,8 @@ return else: self.log.info("%s..." % title) + debug_start('translation-task') + debug_print('starting', goal) self.timer.start_event(goal) try: instrument = False @@ -300,11 +295,13 @@ assert False, 'we should not get here' finally: try: + debug_stop('translation-task') self.timer.end_event(goal) except (KeyboardInterrupt, SystemExit): raise except: pass + #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal) return res def task_annotate(self): @@ -399,7 +396,6 @@ # from pypy.jit.metainterp.warmspot import apply_jit apply_jit(self.translator, policy=self.jitpolicy, - debug_level=JIT_DEBUG[self.config.translation.jit_debug], backend_name=self.config.translation.jit_backend, inline=True) # self.log.info("the JIT compiler was generated") @@ -417,7 +413,6 @@ # from pypy.jit.metainterp.warmspot import apply_jit apply_jit(self.translator, policy=self.jitpolicy, - debug_level=JIT_DEBUG[self.config.translation.jit_debug], backend_name='cli', inline=True) #XXX # self.log.info("the JIT compiler was generated") Modified: pypy/branch/out-of-line-guards/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/goal/app_main.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/goal/app_main.py Thu Dec 9 16:33:59 2010 @@ -11,6 +11,7 @@ -h, --help show this help message and exit -m library module to be run as a script (terminates option list) -W arg warning control (arg is action:message:category:module:lineno) + -E ignore environment variables (such as PYTHONPATH) --version print the PyPy version --info print translation information about this PyPy executable """ @@ -199,7 +200,7 @@ break # found! return newpath -def setup_initial_paths(executable, nanos): +def setup_initial_paths(executable, nanos, readenv=True, **extra): # a substituted os if we are translated global os os = nanos @@ -220,7 +221,7 @@ sys.executable = os.path.abspath(executable) newpath = get_library_path(executable) - path = os.getenv('PYTHONPATH') + path = readenv and os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath # remove duplicates @@ -230,7 +231,6 @@ if dir not in _seen: sys.path.append(dir) _seen[dir] = True - return executable def parse_command_line(argv): @@ -242,6 +242,7 @@ run_stdin = False warnoptions = [] unbuffered = False + readenv = True while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -253,6 +254,8 @@ argv[i] = '-c' run_command = True break + elif arg == '-E': + readenv = False elif arg == '-u': unbuffered = True elif arg == '-O' or arg == '-OO': @@ -305,11 +308,13 @@ run_stdin, warnoptions, unbuffered, + readenv, cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 - # but we need more in the translated PyPy for the compiler package - sys.setrecursionlimit(5000) + # but we need more in the translated PyPy for the compiler package + if '__pypy__' not in sys.builtin_module_names: + sys.setrecursionlimit(5000) if unbuffered: set_unbuffered_io() @@ -355,7 +360,7 @@ # * PYTHONINSPECT is set and stdin is a tty. # return (go_interactive or - (os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) + (readenv and os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) success = True @@ -388,7 +393,7 @@ # If stdin is a tty or if "-i" is specified, we print # a banner and run $PYTHONSTARTUP. print_banner() - python_startup = os.getenv('PYTHONSTARTUP') + python_startup = readenv and os.getenv('PYTHONSTARTUP') if python_startup: try: f = open(python_startup) @@ -451,7 +456,6 @@ '"license" for more information.') def entry_point(executable, argv, nanos): - executable = setup_initial_paths(executable, nanos) try: cmdline = parse_command_line(argv) except CommandLineError, e: @@ -459,8 +463,8 @@ return 2 if cmdline is None: return 0 - else: - return run_command_line(**cmdline) + setup_initial_paths(executable, nanos, **cmdline) + return run_command_line(**cmdline) if __name__ == '__main__': @@ -495,13 +499,13 @@ sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path os = nanos.os_module_for_testing - sys.ps1 = '>>>> ' - sys.ps2 = '.... ' try: sys.exit(int(entry_point(sys.argv[0], sys.argv[1:], os))) finally: - sys.ps1 = '>>> ' # restore the normal ones, in case - sys.ps2 = '... ' # we are dropping to CPython's prompt + # restore the normal prompt (which was changed by _pypy_interact), in + # case we are dropping to CPython's prompt + sys.ps1 = '>>> ' + sys.ps2 = '... ' import os; os.environ.update(reset) assert old_argv is sys.argv assert old_path is sys.path Modified: pypy/branch/out-of-line-guards/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/goal/test2/test_app_main.py Thu Dec 9 16:33:59 2010 @@ -217,6 +217,38 @@ finally: os.environ['PYTHONSTARTUP'] = old + def test_ignore_python_startup(self): + old = os.environ.get('PYTHONSTARTUP', '') + try: + os.environ['PYTHONSTARTUP'] = crashing_demo_script + child = self.spawn(['-E']) + child.expect(re.escape(banner)) + index = child.expect(['Traceback', '>>> ']) + assert index == 1 # no traceback + finally: + os.environ['PYTHONSTARTUP'] = old + + def test_ignore_python_inspect(self): + os.environ['PYTHONINSPECT_'] = '1' + try: + child = self.spawn(['-E', '-c', 'pass']) + from pexpect import EOF + index = child.expect(['>>> ', EOF]) + assert index == 1 # no prompt + finally: + del os.environ['PYTHONINSPECT_'] + + def test_ignore_python_path(self): + old = os.environ.get('PYTHONPATH', '') + try: + os.environ['PYTHONPATH'] = 'foobarbaz' + child = self.spawn(['-E', '-c', 'import sys; print sys.path']) + from pexpect import EOF + index = child.expect(['foobarbaz', EOF]) + assert index == 1 # no foobarbaz + finally: + os.environ['PYTHONPATH'] = old + def test_unbuffered(self): line = 'import os,sys;sys.stdout.write(str(789));os.read(0,1)' child = self.spawn(['-u', '-c', line]) @@ -329,6 +361,10 @@ child = self.spawn(['-mpypy.translator.goal.test2.mymodule']) child.expect('mymodule running') + def test_ps1_only_if_interactive(self): + argv = ['-c', 'import sys; print hasattr(sys, "ps1")'] + child = self.spawn(argv) + child.expect('False') class TestNonInteractive: @@ -534,7 +570,8 @@ newpath = app_main.get_library_path('/tmp/pypy-c') # stdlib not found assert newpath == sys.path newpath = app_main.get_library_path(self.fake_exe) - assert newpath == self.expected_path + # we get at least 'expected_path', and maybe more (e.g.plat-linux2) + assert newpath[:len(self.expected_path)] == self.expected_path finally: sys.path.pop() @@ -547,7 +584,9 @@ app_main.os = os pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') newpath = app_main.get_library_path(pypy_c) - assert len(newpath) == 3 + # we get at least lib_pypy, lib-python/modified-X.Y.Z, + # lib-python/X.Y.Z, and maybe more (e.g. plat-linux2) + assert len(newpath) >= 3 for p in newpath: assert p.startswith(self.trunkdir) finally: Modified: pypy/branch/out-of-line-guards/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/platform/__init__.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/platform/__init__.py Thu Dec 9 16:33:59 2010 @@ -123,7 +123,9 @@ errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() for line in stderrlines: - log.ERROR(line) + log.Error(line) + # ^^^ don't use ERROR, because it might actually be fine. + # Also, ERROR confuses lib-python/conftest.py. raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): @@ -215,13 +217,13 @@ host_factory = Darwin_i386 else: host_factory = Darwin_x86_64 -elif sys.platform == 'freebsd7': - from pypy.translator.platform.freebsd7 import Freebsd7, Freebsd7_64 +elif "freebsd" in sys.platform: + from pypy.translator.platform.freebsd import Freebsd, Freebsd_64 import platform if platform.architecture()[0] == '32bit': - host_factory = Freebsd7 + host_factory = Freebsd else: - host_factory = Freebsd7_64 + host_factory = Freebsd_64 elif os.name == 'nt': from pypy.translator.platform.windows import Windows host_factory = Windows Modified: pypy/branch/out-of-line-guards/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/platform/darwin.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/platform/darwin.py Thu Dec 9 16:33:59 2010 @@ -11,13 +11,17 @@ shared_only = () so_ext = 'so' - + + # NOTE: GCC 4.2 will fail at runtime due to subtle issues, possibly + # related to GC roots. Using LLVM-GCC or Clang will break the build. + default_cc = 'gcc-4.0' + def __init__(self, cc=None): if cc is None: try: cc = os.environ['CC'] except KeyError: - cc = 'gcc' + cc = self.default_cc self.cc = cc def _args_for_shared(self, args): Modified: pypy/branch/out-of-line-guards/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/platform/linux.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/platform/linux.py Thu Dec 9 16:33:59 2010 @@ -27,6 +27,8 @@ class Linux(BaseLinux): + shared_only = () # it seems that on 32-bit linux, compiling with -fPIC + # gives assembler that asmgcc is not happy about. def library_dirs_for_libffi_a(self): # places where we need to look for libffi.a return self.library_dirs_for_libffi() + ['/usr/lib'] Modified: pypy/branch/out-of-line-guards/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/platform/posix.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/platform/posix.py Thu Dec 9 16:33:59 2010 @@ -104,12 +104,10 @@ else: target_name = exe_name.basename - cflags = self.cflags - if sys.maxint > 2147483647: # XXX XXX XXX sort this out - if shared: - cflags = self.cflags + self.shared_only - else: - cflags = self.cflags + self.standalone_only + if shared: + cflags = self.cflags + self.shared_only + else: + cflags = self.cflags + self.standalone_only m = GnuMakefile(path) m.exe_name = exe_name Modified: pypy/branch/out-of-line-guards/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/sandbox/test/test_sandbox.py Thu Dec 9 16:33:59 2010 @@ -148,6 +148,7 @@ if sys.platform == 'linux2': # on Mac, uses another (sandboxsafe) approach expect(f, g, "ll_os.ll_os_open", ("/proc/cpuinfo", 0, 420), OSError(5232, "xyz")) + expect(f, g, "ll_os.ll_os_getenv", ("PYPY_GC_DEBUG",), None) g.close() tail = f.read() f.close() Modified: pypy/branch/out-of-line-guards/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/tool/cbuild.py Thu Dec 9 16:33:59 2010 @@ -309,6 +309,7 @@ #define _POSIX_C_SOURCE 200112L /* Define on FreeBSD to activate all library features */ #define __BSD_VISIBLE 1 +#define __XSI_VISIBLE 700 /* Windows: winsock/winsock2 mess */ #define WIN32_LEAN_AND_MEAN ''' Modified: pypy/branch/out-of-line-guards/pypy/translator/tool/reftracker.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/translator/tool/reftracker.py (original) +++ pypy/branch/out-of-line-guards/pypy/translator/tool/reftracker.py Thu Dec 9 16:33:59 2010 @@ -3,7 +3,7 @@ Usage: call track(obj). """ -import autopath, sys, os +import autopath, sys, os, types import gc from pypy.translator.tool.graphpage import GraphPage, DotGen from pypy.tool.uid import uid @@ -39,7 +39,7 @@ if o2 is None: continue addedge(objectlist[i], o2) - id2typename[uid(o2)] = type(o2).__name__ + id2typename[uid(o2)] = self.shortrepr(o2) del o2 for o2 in self.get_referrers(objectlist[i]): if o2 is None: @@ -47,7 +47,7 @@ if type(o2) is list and o2 and o2[0] is MARKER: continue addedge(o2, objectlist[i]) - id2typename[uid(o2)] = type(o2).__name__ + id2typename[uid(o2)] = self.shortrepr(o2) del o2 for ids, label in edges.items(): @@ -82,13 +82,23 @@ return self.newpage(objectlist) def formatobject(self, o): + header = self.shortrepr(o, compact=False) + secondline = repr(o.__class__) + return header, secondline, repr(o) + + def shortrepr(self, o, compact=True): + t = type(o) + if t is types.FrameType: + if compact: + return 'frame %r' % (o.f_code.co_name,) + else: + return 'frame %r' % (o.f_code,) s = repr(o) if len(s) > 50: - linktext = s s = s[:20] + ' ... ' + s[-20:] - else: - linktext = '' - return type(o).__name__, s, linktext + if s.startswith('<') and s.endswith('>'): + s = s[1:-1] + return s def edgelabel(self, o1, o2): return '' From fijal at codespeak.net Thu Dec 9 17:07:15 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Thu, 9 Dec 2010 17:07:15 +0100 (CET) Subject: [pypy-svn] r79936 - in pypy/branch/out-of-line-guards/pypy: jit/backend/llgraph jit/codewriter jit/codewriter/test jit/metainterp jit/metainterp/test rpython/lltypesystem Message-ID: <20101209160715.8A2C6282B9D@codespeak.net> Author: fijal Date: Thu Dec 9 17:07:12 2010 New Revision: 79936 Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/call.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/effectinfo.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/blackhole.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/resoperation.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/lloperation.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py Log: Try to merge my changes up-to-date. I'm not completely sure, since SVN screwed me up massively on this one Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py Thu Dec 9 17:07:12 2010 @@ -157,6 +157,7 @@ 'force_token' : ((), 'int'), 'call_may_force' : (('int', 'varargs'), 'intorptr'), 'guard_not_forced': ((), None), + 'guard_not_invariant': ((), None), } # ____________________________________________________________ @@ -925,6 +926,8 @@ if forced: raise GuardFailed + def op_guard_not_invariant(self, descr): + pass class OOFrame(Frame): Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py Thu Dec 9 17:07:12 2010 @@ -488,6 +488,10 @@ self.latest_frame = frame return self.get_fail_descr_from_number(fail_index) + def get_invalidate_asm(self, TP, fieldname): + def invalidate_asm(arg, fieldname): + XXX # write me + return invalidate_asm class OOtypeCPU_xxx_disabled(BaseCPU): is_oo = True Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/call.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/call.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/call.py Thu Dec 9 17:07:12 2010 @@ -6,6 +6,7 @@ from pypy.jit.codewriter import support from pypy.jit.codewriter.jitcode import JitCode from pypy.jit.codewriter.effectinfo import VirtualizableAnalyzer +from pypy.jit.codewriter.effectinfo import JitInvariantAnalyzer from pypy.jit.codewriter.effectinfo import effectinfo_from_writeanalyze from pypy.jit.codewriter.effectinfo import EffectInfo, CallInfoCollection from pypy.translator.simplify import get_funcobj, get_functype @@ -30,6 +31,7 @@ self.raise_analyzer = RaiseAnalyzer(translator) self.readwrite_analyzer = ReadWriteAnalyzer(translator) self.virtualizable_analyzer = VirtualizableAnalyzer(translator) + self.jit_invariant_analyzer = JitInvariantAnalyzer(translator) # for index, jd in enumerate(jitdrivers_sd): jd.index = index @@ -216,7 +218,9 @@ assert not NON_VOID_ARGS, ("arguments not supported for " "loop-invariant function!") # build the extraeffect - if self.virtualizable_analyzer.analyze(op): + if self.jit_invariant_analyzer.analyze(op): + extraeffect = EffectInfo.EF_FORCES_JIT_INVARIANT + elif self.virtualizable_analyzer.analyze(op): extraeffect = EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE elif loopinvariant: extraeffect = EffectInfo.EF_LOOPINVARIANT @@ -234,7 +238,7 @@ # if pure or loopinvariant: assert effectinfo is not None - assert extraeffect != EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE + assert extraeffect < EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE # return self.cpu.calldescrof(FUNC, tuple(NON_VOID_ARGS), RESULT, effectinfo) Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/effectinfo.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/effectinfo.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/effectinfo.py Thu Dec 9 17:07:12 2010 @@ -14,6 +14,7 @@ EF_CAN_RAISE = 2 #normal function (can raise) EF_LOOPINVARIANT = 3 #special: call it only once per loop EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 4 #can raise and force virtualizables + EF_FORCES_JIT_INVARIANT = 5 #can force jit invariant # the 'oopspecindex' field is one of the following values: OS_NONE = 0 # normal case, no oopspec @@ -142,6 +143,10 @@ return op.opname in ('jit_force_virtualizable', 'jit_force_virtual') +class JitInvariantAnalyzer(BoolGraphAnalyzer): + def analyze_simple_operation(self, op): + return op.opname == 'jit_invariant_setfield' + # ____________________________________________________________ class CallInfoCollection(object): Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py Thu Dec 9 17:07:12 2010 @@ -32,6 +32,7 @@ self.cpu = cpu self.callcontrol = callcontrol self.portal_jd = portal_jd # non-None only for the portal graph(s) + self.asmcodes_appenders = {} def transform(self, graph): self.graph = graph @@ -554,6 +555,7 @@ arrayfielddescr, arraydescr) return [] + extrargs = [] # check for _immutable_fields_ hints if v_inst.concretetype.TO._immutable_field(c_fieldname.value): if (self.callcontrol is not None and @@ -565,7 +567,10 @@ else: jit_inv = v_inst.concretetype.TO._hints.get('jit_invariant_fields') if jit_inv and op.args[1].value in jit_inv.fields: - pure = '_pure' + pure = '_invariant' + c_func = self._get_appender_for_asmcode( + v_inst.concretetype.TO, c_fieldname.value) + extrargs.append(c_func) else: pure = '' argname = getattr(v_inst.concretetype.TO, '_gckind', 'gc') @@ -573,7 +578,40 @@ c_fieldname.value) kind = getkind(RESULT)[0] return SpaceOperation('getfield_%s_%s%s' % (argname, kind, pure), - [v_inst, descr], op.result) + [v_inst, descr] + extrargs, op.result) + + def _get_appender_for_asmcode(self, TP, fieldname): + from pypy.rpython.lltypesystem.rclass import ASMCODE + from pypy.rpython.annlowlevel import MixLevelHelperAnnotator + from pypy.annotation.model import lltype_to_annotation + + key = (TP, fieldname) + try: + return self.asmcodes_appenders[key] + except KeyError: + name = 'asmcodes_' + fieldname[len('inst_'):] + + def appender(inst, v): + inst = lltype.cast_opaque_ptr(lltype.Ptr(TP), inst) + new_asmcode = lltype.malloc(ASMCODE) + new_asmcode.address = v + next = getattr(inst, name) + if not next: + setattr(inst, name, new_asmcode) + else: + prev = next + while next: + prev = next + next = next.next + prev.next = new_asmcode + + args_s = [lltype_to_annotation(llmemory.GCREF)] * 2 + s_result = lltype_to_annotation(lltype.Void) + mixlevelann = MixLevelHelperAnnotator(self.cpu.rtyper) + c_appender = mixlevelann.constfunc(appender, args_s, s_result) + mixlevelann.finish() + self.asmcodes_appenders[key] = c_appender + return c_appender def rewrite_op_setfield(self, op): if self.is_typeptr_getset(op): Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/test/test_jtransform.py Thu Dec 9 17:07:12 2010 @@ -956,7 +956,8 @@ Constant('inst_x', lltype.Void)], v2) tr = Transformer(FakeCPU()) op1 = tr.rewrite_operation(op) - assert op1.opname == 'getfield_gc_i_pure' + assert op1.opname == 'getfield_gc_i_invariant' + assert len(op1.args) == 3 def test_jit_invariant_setfield(): from pypy.rpython.rclass import FieldListAccessor Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/blackhole.py Thu Dec 9 17:07:12 2010 @@ -1087,6 +1087,10 @@ bhimpl_getfield_gc_r_pure = bhimpl_getfield_gc_r bhimpl_getfield_gc_f_pure = bhimpl_getfield_gc_f + bhimpl_getfield_gc_i_invariant = bhimpl_getfield_gc_i + bhimpl_getfield_gc_r_invariant = bhimpl_getfield_gc_r + bhimpl_getfield_gc_f_invariant = bhimpl_getfield_gc_f + bhimpl_getfield_vable_i = bhimpl_getfield_gc_i bhimpl_getfield_vable_r = bhimpl_getfield_gc_r bhimpl_getfield_vable_f = bhimpl_getfield_gc_f Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py Thu Dec 9 17:07:12 2010 @@ -101,6 +101,7 @@ if old_loop_token is not None: metainterp.staticdata.log("reusing old loop") return old_loop_token + metainterp.remember_jit_invariants(loop) send_loop_to_backend(metainterp_sd, loop, "loop") insert_loop_token(old_loop_tokens, loop_token) record_loop_or_bridge(loop) @@ -563,6 +564,7 @@ # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr) prepare_last_operation(new_loop, target_loop_token) resumekey.compile_and_attach(metainterp, new_loop) + metainterp.remember_jit_invariants(target_loop_token) record_loop_or_bridge(new_loop) return target_loop_token Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py Thu Dec 9 17:07:12 2010 @@ -1,5 +1,6 @@ import py, os, sys from pypy.rpython.lltypesystem import lltype, llmemory, rclass +from pypy.rpython.lltypesystem.rclass import ASMCODE from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -509,6 +510,14 @@ opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any + @arguments("box", "descr", "box") + def _opimpl_getfield_gc_invariant_any(self, box, fielddescr, c_func): + self.metainterp.invariant_structs.append((box, c_func)) + return self.execute_with_descr(rop.GETFIELD_GC_PURE, fielddescr, box) + opimpl_getfield_gc_i_invariant = _opimpl_getfield_gc_invariant_any + opimpl_getfield_gc_r_invariant = _opimpl_getfield_gc_invariant_any + opimpl_getfield_gc_f_invariant = _opimpl_getfield_gc_invariant_any + @arguments("orgpc", "box", "descr") def _opimpl_getfield_gc_greenfield_any(self, pc, box, fielddescr): ginfo = self.metainterp.jitdriver_sd.greenfield_info @@ -1163,10 +1172,9 @@ assert i == len(allboxes) # effectinfo = descr.get_extra_info() - if (effectinfo is None or - effectinfo.extraeffect == - effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE or - assembler_call): + force_vir = effectinfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE + if (effectinfo is None or effectinfo.extraeffect >= force_vir or + assembler_call): # residual calls require attention to keep virtualizables in-sync self.metainterp.clear_exception() self.metainterp.vable_and_vrefs_before_residual_call() @@ -1178,6 +1186,8 @@ if resbox is not None: self.make_result_of_lastop(resbox) self.metainterp.vable_after_residual_call() + if effectinfo.extraeffect >= effectinfo.EF_FORCES_JIT_INVARIANT: + self.generate_guard(rop.GUARD_NOT_INVARIANT, None) self.generate_guard(rop.GUARD_NOT_FORCED, None) self.metainterp.handle_possible_exception() return resbox @@ -1393,6 +1403,7 @@ self.portal_trace_positions = [] self.free_frames_list = [] self.last_exc_value_box = None + self.invariant_structs = [] def perform_call(self, jitcode, boxes, greenkey=None): # causes the metainterp to enter the given subfunction @@ -2227,6 +2238,14 @@ op = op.copy_and_change(rop.CALL_ASSEMBLER, args=args, descr=token) self.history.operations.append(op) + def remember_jit_invariants(self, token): + lltoken = lltype.cast_opaque_ptr(llmemory.GCREF, token) + seen = {} + for b_struct, c_appender in self.invariant_structs: + if (b_struct, c_appender) not in seen: + heaptracker.int2adr(c_func.value).ptr(b_struct.value, lltoken) + seend[(b_struct, c_appender)] = None + # ____________________________________________________________ class GenerateMergePoint(JitException): Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/resoperation.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/resoperation.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/resoperation.py Thu Dec 9 17:07:12 2010 @@ -379,7 +379,8 @@ 'GUARD_EXCEPTION/1d', 'GUARD_NO_OVERFLOW/0d', 'GUARD_OVERFLOW/0d', - 'GUARD_NOT_FORCED/0d', + 'GUARD_NOT_FORCED/0d', # forcing virtualrefs/virtualizables + 'GUARD_NOT_INVARIANT/0d', # forcing jit invariant fields '_GUARD_LAST', # ----- end of guard operations ----- '_NOSIDEEFFECT_FIRST', # ----- start of no_side_effect operations ----- Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py Thu Dec 9 17:07:12 2010 @@ -723,7 +723,7 @@ def test_getfield_jit_invariant(self): class A(object): - _jit_invariant_fields_ = 'x' + _jit_invariant_fields_ = ['x'] a1 = A() a1.x = 5 @@ -739,6 +739,31 @@ res = self.interp_operations(f, [-3]) self.check_operations_history(getfield_gc = 0) + def test_setfield_jit_invariant(self): + class A(object): + _jit_invariant_fields_ = ['x'] + + a = A() + a.x = 1 + + myjitdriver = JitDriver(greens = [], reds = ['i']) + + @dont_look_inside + def g(i): + if i == 5: + a.x = 5 + + def f(): + i = 0 + while i < 10: + myjitdriver.can_enter_jit(i=i) + myjitdriver.jit_merge_point(i=i) + g(i) + i += a.x + + self.meta_interp(f, []) + self.check_loop_count(2) + def test_setfield_bool(self): class A: def __init__(self): Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmspot.py Thu Dec 9 17:07:12 2010 @@ -820,10 +820,19 @@ op.opname = 'direct_call' op.args[:3] = [closures[key]] + def replace_jit_invariant_with_direct_call(self, op): + op.opname = 'direct_call' + ARG = op.args[0].concretetype + FUNC = lltype.Ptr(lltype.FuncType([ARG, lltype.Void], lltype.Void)) + llptr = self.helper_func(FUNC, self.cpu.get_invalidate_asm(FUNC, + op.args[1].value)) + cptr = Constant(llptr, lltype.Void) + op.args = [cptr, op.args[0], op.args[1]] + def rewrite_jit_invariant_setfield(self): graphs = self.translator.graphs for graph, block, i in find_jit_invariant_setfield(graphs): - del block.operations[i] # for now + self.replace_jit_invariant_with_direct_call(block.operations[i]) def rewrite_force_virtual(self, vrefinfo): if self.cpu.ts.name != 'lltype': Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/lloperation.py Thu Dec 9 17:07:12 2010 @@ -433,6 +433,7 @@ 'jit_marker': LLOp(), 'jit_force_virtualizable':LLOp(canrun=True), 'jit_force_virtual': LLOp(canrun=True), + 'jit_invariant_setfield': LLOp(canrun=True), 'get_exception_addr': LLOp(), 'get_exc_value_addr': LLOp(), 'do_malloc_fixedsize_clear':LLOp(canraise=(MemoryError,),canunwindgc=True), Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py Thu Dec 9 17:07:12 2010 @@ -87,7 +87,7 @@ # a linked-list of assembler codes to invalidate in case jit_invariant_fields # are modified ASMCODE = lltype.GcForwardReference() -ASMCODE.become(GcStruct('asmcode', ('address', llmemory.Address), +ASMCODE.become(GcStruct('asmcode', ('address', llmemory.GCREF), ('next', lltype.Ptr(ASMCODE)))) def cast_vtable_to_typeptr(vtable): @@ -148,6 +148,9 @@ # self.rbase = getclassrepr(self.rtyper, self.classdef.basedef) self.rbase.setup() + # jit_invariant_fields + jit_inv_fields = self.classdef.classdesc.classdict.get( + '_jit_invariant_fields_') kwds = {'hints': {'immutable': True}} vtable_type = Struct('%s_vtable' % self.classdef.name, ('super', self.rbase.vtable_type), @@ -517,7 +520,9 @@ # for virtualizables; see rvirtualizable2.py if (op == 'setfield' and cname.value.startswith('inst_') and cname.value[len('inst_'):] in self.jit_invariant_fields): - llops.genop('jit_invariant_setfield', []) + name = cname.value + cname = Constant('asmcodes_' + name[len('inst_'):], lltype.Void) + llops.genop('jit_invariant_setfield', [vinst, cname]) def new_instance(self, llops, classcallhop=None): """Build a new instance, without calling __init__.""" From danchr at codespeak.net Thu Dec 9 17:50:23 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Thu, 9 Dec 2010 17:50:23 +0100 (CET) Subject: [pypy-svn] r79937 - pypy/extradoc/planning Message-ID: <20101209165023.939C7282B9D@codespeak.net> Author: danchr Date: Thu Dec 9 17:50:19 2010 New Revision: 79937 Modified: pypy/extradoc/planning/1.4.1.txt Log: Remove slash from "Mac OS X" Modified: pypy/extradoc/planning/1.4.1.txt ============================================================================== --- pypy/extradoc/planning/1.4.1.txt (original) +++ pypy/extradoc/planning/1.4.1.txt Thu Dec 9 17:50:19 2010 @@ -11,7 +11,7 @@ * Fix a corner case in the GC (minimark). -* Fix cpyext on Mac OS/X. (Loading C extension modules in PyPy is still +* Fix cpyext on Mac OS X. (Loading C extension modules in PyPy is still considered *alpha stage!*) * Fixed a corner case in the JIT, leading to "Fatal RPython error: From arigo at codespeak.net Thu Dec 9 18:30:35 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 18:30:35 +0100 (CET) Subject: [pypy-svn] r79938 - in pypy/trunk/pypy: module/posix module/posix/test rpython/module translator/c/test Message-ID: <20101209173035.299A0282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 18:30:33 2010 New Revision: 79938 Modified: pypy/trunk/pypy/module/posix/__init__.py pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py pypy/trunk/pypy/rpython/module/ll_os.py pypy/trunk/pypy/translator/c/test/test_extfunc.py Log: Implement os.getloadavg(). (Phew, we really need to edit files in all corners of the world for this...) Modified: pypy/trunk/pypy/module/posix/__init__.py ============================================================================== --- pypy/trunk/pypy/module/posix/__init__.py (original) +++ pypy/trunk/pypy/module/posix/__init__.py Thu Dec 9 18:30:33 2010 @@ -111,6 +111,8 @@ interpleveldefs['sysconf_names'] = 'space.wrap(os.sysconf_names)' if hasattr(os, 'ttyname'): interpleveldefs['ttyname'] = 'interp_posix.ttyname' + if hasattr(os, 'getloadavg'): + interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Thu Dec 9 18:30:33 2010 @@ -963,6 +963,17 @@ return space.w_None chown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] +def getloadavg(space): + try: + load = os.getloadavg() + except OSError, e: + raise OperationError(space.w_OSError, + space.wrap("Load averages are unobtainable")) + return space.newtuple([space.wrap(load[0]), + space.wrap(load[1]), + space.wrap(load[2])]) +getloadavg.unwrap_spec = [ObjSpace] + if _WIN: from pypy.rlib import rwin32 Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Thu Dec 9 18:30:33 2010 @@ -521,6 +521,14 @@ assert os.WIFEXITED(status) assert os.WEXITSTATUS(status) == exit_status + if hasattr(os, 'getloadavg'): + def test_os_getloadavg(self): + os = self.posix + l0, l1, l2 = os.getloadavg() + assert type(l0) is float and l0 >= 0.0 + assert type(l1) is float and l0 >= 0.0 + assert type(l2) is float and l0 >= 0.0 + if hasattr(os, 'fsync'): def test_fsync(self): os = self.posix Modified: pypy/trunk/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_os.py (original) +++ pypy/trunk/pypy/rpython/module/ll_os.py Thu Dec 9 18:30:33 2010 @@ -730,6 +730,22 @@ return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) + @registering_if(os, 'getloadavg') + def register_os_getloadavg(self): + AP = rffi.CArrayPtr(lltype.Float) + c_getloadavg = self.llexternal('getloadavg', [AP, rffi.INT], rffi.INT) + + def getloadavg_llimpl(): + load = lltype.malloc(AP.TO, 3, flavor='raw') + r = c_getloadavg(load, 3) + result_tuple = load[0], load[1], load[2] + lltype.free(load, flavor='raw') + if r != 3: + raise OSError + return result_tuple + return extdef([], (float, float, float), + "ll_os.ll_getloadavg", llimpl=getloadavg_llimpl) + # ------------------------------- os.read ------------------------------- @registering(os.read) Modified: pypy/trunk/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_extfunc.py Thu Dec 9 18:30:33 2010 @@ -755,3 +755,13 @@ for i in range(5): res = func(i) assert res == os.uname()[i] + +if hasattr(os, 'getloadavg'): + def test_os_getloadavg(): + def does_stuff(): + a, b, c = os.getloadavg() + print a, b, c + return a + b + c + f = compile(does_stuff, []) + res = f() + assert type(res) is float and res >= 0.0 From arigo at codespeak.net Thu Dec 9 18:43:48 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 18:43:48 +0100 (CET) Subject: [pypy-svn] r79939 - in pypy/trunk/pypy/module/posix: . test Message-ID: <20101209174348.6B09D282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 18:43:46 2010 New Revision: 79939 Modified: pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py Log: Support os.fsync(f) and os.fdatasync(f), where f is any object with a fileno() method. Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Thu Dec 9 18:43:46 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 +from pypy.interpreter.error import operationerrfmt from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -151,19 +152,34 @@ raise wrap_oserror(space, e) ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] -def fsync(space, fd): +def _as_filedescriptor(space, w_fd): + try: + fd = space.c_int_w(w_fd) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + w_fd = space.call_method(w_fd, 'fileno') + fd = space.c_int_w(w_fd) + if fd < 0: + raise operationerrfmt(space.w_ValueError, + "file descriptor cannot be a negative integer (%d)", fd) + return fd + +def fsync(space, w_fd): + fd = _as_filedescriptor(space, w_fd) try: os.fsync(fd) except OSError, e: raise wrap_oserror(space, e) -fsync.unwrap_spec = [ObjSpace, "c_int"] +fsync.unwrap_spec = [ObjSpace, W_Root] -def fdatasync(space, fd): +def fdatasync(space, w_fd): + fd = _as_filedescriptor(space, w_fd) try: os.fdatasync(fd) except OSError, e: raise wrap_oserror(space, e) -fdatasync.unwrap_spec = [ObjSpace, "c_int"] +fdatasync.unwrap_spec = [ObjSpace, W_Root] # ____________________________________________________________ Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Thu Dec 9 18:43:46 2010 @@ -536,14 +536,11 @@ try: fd = f.fileno() os.fsync(fd) - finally: + os.fsync(f) # <- should also work with a file, or anything + finally: # with a fileno() method f.close() - try: - os.fsync(fd) - except OSError: - pass - else: - raise AssertionError("os.fsync didn't raise") + raises(OSError, os.fsync, fd) + raises(ValueError, os.fsync, -1) if hasattr(os, 'fdatasync'): def test_fdatasync(self): @@ -554,12 +551,8 @@ os.fdatasync(fd) finally: f.close() - try: - os.fdatasync(fd) - except OSError: - pass - else: - raise AssertionError("os.fdatasync didn't raise") + raises(OSError, os.fdatasync, fd) + raises(ValueError, os.fdatasync, -1) def test_largefile(self): os = self.posix From arigo at codespeak.net Thu Dec 9 18:53:54 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 18:53:54 +0100 (CET) Subject: [pypy-svn] r79940 - in pypy/trunk/pypy: interpreter module/posix module/posix/test module/select Message-ID: <20101209175354.79BC5282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 18:53:52 2010 New Revision: 79940 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py pypy/trunk/pypy/module/select/interp_select.py Log: Thanks agaynor for pointing out that the select module already had a similar function. Combine both versions and move it as space.c_filedescriptor_w(). Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Thu Dec 9 18:53:52 2010 @@ -1183,6 +1183,27 @@ self.wrap("expected a 32-bit integer")) return value + def c_filedescriptor_w(self, w_fd): + try: + fd = self.c_int_w(w_fd) + except OperationError, e: + if not e.match(self, self.w_TypeError): + raise + try: + w_fileno = self.getattr(w_fd, self.wrap('fileno')) + except OperationError, e: + if e.match(self, self.w_AttributeError): + raise OperationError(self.w_TypeError, + self.wrap("argument must be an int, " + "or have a fileno() method.")) + raise + w_fd = self.call_function(w_fileno) + fd = self.c_int_w(w_fd) + if fd < 0: + raise operationerrfmt(self.w_ValueError, + "file descriptor cannot be a negative integer (%d)", fd) + return fd + def warn(self, msg, w_warningcls): self.appexec([self.wrap(msg), w_warningcls], """(msg, warningcls): import warnings Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Thu Dec 9 18:53:52 2010 @@ -152,21 +152,8 @@ raise wrap_oserror(space, e) ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] -def _as_filedescriptor(space, w_fd): - try: - fd = space.c_int_w(w_fd) - except OperationError, e: - if not e.match(space, space.w_TypeError): - raise - w_fd = space.call_method(w_fd, 'fileno') - fd = space.c_int_w(w_fd) - if fd < 0: - raise operationerrfmt(space.w_ValueError, - "file descriptor cannot be a negative integer (%d)", fd) - return fd - def fsync(space, w_fd): - fd = _as_filedescriptor(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) try: os.fsync(fd) except OSError, e: @@ -174,7 +161,7 @@ fsync.unwrap_spec = [ObjSpace, W_Root] def fdatasync(space, w_fd): - fd = _as_filedescriptor(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) try: os.fdatasync(fd) except OSError, e: Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Thu Dec 9 18:53:52 2010 @@ -536,6 +536,7 @@ try: fd = f.fileno() os.fsync(fd) + os.fsync(long(fd)) os.fsync(f) # <- should also work with a file, or anything finally: # with a fileno() method f.close() Modified: pypy/trunk/pypy/module/select/interp_select.py ============================================================================== --- pypy/trunk/pypy/module/select/interp_select.py (original) +++ pypy/trunk/pypy/module/select/interp_select.py Thu Dec 9 18:53:52 2010 @@ -12,37 +12,17 @@ unregistering file descriptors, and then polling them for I/O events.""" return Poll() -def as_fd_w(space, w_fd): - if not space.is_true(space.isinstance(w_fd, space.w_int)): - try: - w_fileno = space.getattr(w_fd, space.wrap('fileno')) - except OperationError, e: - if e.match(space, space.w_AttributeError): - raise OperationError(space.w_TypeError, - space.wrap("argument must be an int, or have a fileno() method.")) - raise - w_fd = space.call_function(w_fileno) - if not space.is_true(space.isinstance(w_fd, space.w_int)): - raise OperationError(space.w_TypeError, - space.wrap('filneo() return a non-integer')) - - fd = space.int_w(w_fd) - if fd < 0: - raise operationerrfmt(space.w_ValueError, - "file descriptor cannot be a negative integer (%d)", fd) - return fd - class Poll(Wrappable): def __init__(self): self.fddict = {} def register(self, space, w_fd, events=defaultevents): - fd = as_fd_w(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) self.fddict[fd] = events register.unwrap_spec = ['self', ObjSpace, W_Root, int] def unregister(self, space, w_fd): - fd = as_fd_w(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) try: del self.fddict[fd] except KeyError: @@ -113,9 +93,9 @@ iwtd_w = space.listview(w_iwtd) owtd_w = space.listview(w_owtd) ewtd_w = space.listview(w_ewtd) - iwtd = [as_fd_w(space, w_f) for w_f in iwtd_w] - owtd = [as_fd_w(space, w_f) for w_f in owtd_w] - ewtd = [as_fd_w(space, w_f) for w_f in ewtd_w] + iwtd = [space.c_filedescriptor_w(w_f) for w_f in iwtd_w] + owtd = [space.c_filedescriptor_w(w_f) for w_f in owtd_w] + ewtd = [space.c_filedescriptor_w(w_f) for w_f in ewtd_w] iwtd_d = {} owtd_d = {} ewtd_d = {} From arigo at codespeak.net Thu Dec 9 19:03:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 19:03:51 +0100 (CET) Subject: [pypy-svn] r79941 - in pypy/trunk/pypy: module/posix module/posix/test rpython/module translator/c/test Message-ID: <20101209180351.69C2C282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 19:03:49 2010 New Revision: 79941 Modified: pypy/trunk/pypy/module/posix/__init__.py pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py pypy/trunk/pypy/rpython/module/ll_os.py pypy/trunk/pypy/translator/c/test/test_extfunc.py Log: os.fchdir(). Modified: pypy/trunk/pypy/module/posix/__init__.py ============================================================================== --- pypy/trunk/pypy/module/posix/__init__.py (original) +++ pypy/trunk/pypy/module/posix/__init__.py Thu Dec 9 19:03:49 2010 @@ -77,6 +77,8 @@ interpleveldefs['fsync'] = 'interp_posix.fsync' if hasattr(os, 'fdatasync'): interpleveldefs['fdatasync'] = 'interp_posix.fdatasync' + if hasattr(os, 'fchdir'): + interpleveldefs['fchdir'] = 'interp_posix.fchdir' if hasattr(os, 'putenv'): interpleveldefs['putenv'] = 'interp_posix.putenv' if hasattr(posix, 'unsetenv'): # note: emulated in os Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Thu Dec 9 19:03:49 2010 @@ -168,6 +168,14 @@ raise wrap_oserror(space, e) fdatasync.unwrap_spec = [ObjSpace, W_Root] +def fchdir(space, w_fd): + fd = space.c_filedescriptor_w(w_fd) + try: + os.fchdir(fd) + except OSError, e: + raise wrap_oserror(space, e) +fchdir.unwrap_spec = [ObjSpace, W_Root] + # ____________________________________________________________ # For LL backends, expose all fields. Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Thu Dec 9 19:03:49 2010 @@ -546,7 +546,7 @@ if hasattr(os, 'fdatasync'): def test_fdatasync(self): os = self.posix - f = open(self.path2) + f = open(self.path2, "w") try: fd = f.fileno() os.fdatasync(fd) @@ -555,6 +555,24 @@ raises(OSError, os.fdatasync, fd) raises(ValueError, os.fdatasync, -1) + if hasattr(os, 'fchdir'): + def test_fchdir(self): + os = self.posix + localdir = os.getcwd() + try: + os.mkdir(self.path2 + 'dir') + fd = os.open(self.path2 + 'dir', os.O_RDONLY) + try: + os.fchdir(fd) + mypath = os.getcwd() + finally: + os.close(fd) + assert mypath.endswith('test_posix2-dir') + raises(OSError, os.fchdir, fd) + raises(ValueError, os.fchdir, -1) + finally: + os.chdir(localdir) + def test_largefile(self): os = self.posix fd = os.open(self.path2 + 'test_largefile', os.O_RDWR | os.O_CREAT, 0666) Modified: pypy/trunk/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_os.py (original) +++ pypy/trunk/pypy/rpython/module/ll_os.py Thu Dec 9 19:03:49 2010 @@ -903,6 +903,18 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") + @registering_if(os, 'fchdir') + def register_os_fchdir(self): + os_fchdir = self.llexternal('fchdir', [rffi.INT], rffi.INT) + + def fchdir_llimpl(fd): + res = rffi.cast(rffi.LONG, os_fchdir(rffi.cast(rffi.INT, fd))) + if res < 0: + raise OSError(rposix.get_errno(), "fchdir failed") + return extdef([int], s_None, + llimpl=fchdir_llimpl, + export_name="ll_os.ll_os_fchdir") + @registering_str_unicode(os.access) def register_os_access(self, traits): os_access = self.llexternal(traits.posix_function_name('access'), Modified: pypy/trunk/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_extfunc.py Thu Dec 9 19:03:49 2010 @@ -765,3 +765,21 @@ f = compile(does_stuff, []) res = f() assert type(res) is float and res >= 0.0 + +if hasattr(os, 'fchdir'): + def test_os_fchdir(): + def does_stuff(): + fd = os.open('/', os.O_RDONLY, 0400) + try: + os.fchdir(fd) + s = os.getcwd() + finally: + os.close(fd) + return s == '/' + f = compile(does_stuff, []) + localdir = os.getcwd() + try: + res = f() + finally: + os.chdir(localdir) + assert res == True From arigo at codespeak.net Thu Dec 9 19:14:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 19:14:21 +0100 (CET) Subject: [pypy-svn] r79942 - pypy/extradoc/planning Message-ID: <20101209181421.6AEA5282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 19:14:19 2010 New Revision: 79942 Modified: pypy/extradoc/planning/1.4.1.txt Log: Add to planning. Modified: pypy/extradoc/planning/1.4.1.txt ============================================================================== --- pypy/extradoc/planning/1.4.1.txt (original) +++ pypy/extradoc/planning/1.4.1.txt Thu Dec 9 19:14:19 2010 @@ -25,3 +25,9 @@ * Merge jit-unroll-loops (?) * Migrate to mercurial + +* Finish looking in the ctypes bug:: + + c = cast(cast(c_char_p("hello"), c_void_p), c_char_p) + gc.collect() + c.value # garbage From arigo at codespeak.net Thu Dec 9 19:17:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Thu, 9 Dec 2010 19:17:01 +0100 (CET) Subject: [pypy-svn] r79943 - pypy/extradoc/planning Message-ID: <20101209181701.5A062282B9D@codespeak.net> Author: arigo Date: Thu Dec 9 19:16:59 2010 New Revision: 79943 Modified: pypy/extradoc/planning/1.4.1.txt Log: Already did some of that, and more is planned. Modified: pypy/extradoc/planning/1.4.1.txt ============================================================================== --- pypy/extradoc/planning/1.4.1.txt (original) +++ pypy/extradoc/planning/1.4.1.txt Thu Dec 9 19:16:59 2010 @@ -17,6 +17,8 @@ * Fixed a corner case in the JIT, leading to "Fatal RPython error: AssertionError". +* Added some missing functions from the 'os' module. + Plan: From agaynor at codespeak.net Thu Dec 9 19:27:51 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Thu, 9 Dec 2010 19:27:51 +0100 (CET) Subject: [pypy-svn] r79944 - in pypy/trunk/pypy/interpreter: . test Message-ID: <20101209182751.0B48F282B9D@codespeak.net> Author: agaynor Date: Thu Dec 9 19:27:50 2010 New Revision: 79944 Modified: pypy/trunk/pypy/interpreter/function.py pypy/trunk/pypy/interpreter/test/test_function.py Log: Let the other side of __eq__ be called for comparison of methods with other types. Modified: pypy/trunk/pypy/interpreter/function.py ============================================================================== --- pypy/trunk/pypy/interpreter/function.py (original) +++ pypy/trunk/pypy/interpreter/function.py Thu Dec 9 19:27:50 2010 @@ -48,7 +48,7 @@ return "<%s %s>" % (self.__class__.__name__, name) def call_args(self, args): - # delegate activation to code + # delegate activation to code return self.getcode().funcrun(self, args) def call_obj_args(self, w_obj, args): @@ -61,17 +61,17 @@ return _get_immutable_code(self) return jit.hint(self.code, promote=True) return self.code - + def funccall(self, *args_w): # speed hack from pypy.interpreter import gateway from pypy.interpreter.pycode import PyCode - + code = self.getcode() # hook for the jit nargs = len(args_w) fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: if nargs == 0: - assert isinstance(code, gateway.BuiltinCode0) + assert isinstance(code, gateway.BuiltinCode0) return code.fastcall_0(self.space, self) elif nargs == 1: assert isinstance(code, gateway.BuiltinCode1) @@ -80,22 +80,22 @@ assert isinstance(code, gateway.BuiltinCode2) return code.fastcall_2(self.space, self, args_w[0], args_w[1]) elif nargs == 3: - assert isinstance(code, gateway.BuiltinCode3) + assert isinstance(code, gateway.BuiltinCode3) return code.fastcall_3(self.space, self, args_w[0], args_w[1], args_w[2]) elif nargs == 4: - assert isinstance(code, gateway.BuiltinCode4) + assert isinstance(code, gateway.BuiltinCode4) return code.fastcall_4(self.space, self, args_w[0], args_w[1], args_w[2], args_w[3]) elif (nargs|PyCode.FLATPYCALL) == fast_natural_arity: - assert isinstance(code, PyCode) + assert isinstance(code, PyCode) if nargs < 5: new_frame = self.space.createframe(code, self.w_func_globals, self.closure) for i in funccallunrolling: if i < nargs: new_frame.fastlocals_w[i] = args_w[i] - return new_frame.run() + return new_frame.run() elif nargs >= 1 and fast_natural_arity == Code.PASSTHROUGHARGS1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) return code.funcrun_obj(self, args_w[0], @@ -106,9 +106,9 @@ def funccall_valuestack(self, nargs, frame): # speed hack from pypy.interpreter import gateway from pypy.interpreter.pycode import PyCode - + code = self.getcode() # hook for the jit - fast_natural_arity = code.fast_natural_arity + fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: if nargs == 0: assert isinstance(code, gateway.BuiltinCode0) @@ -143,7 +143,7 @@ w_obj = frame.peekvalue(nargs-1) args = frame.make_arguments(nargs-1) return code.funcrun_obj(self, w_obj, args) - + args = frame.make_arguments(nargs) return self.call_args(args) @@ -155,8 +155,8 @@ for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) new_frame.fastlocals_w[i] = w_arg - - return new_frame.run() + + return new_frame.run() @jit.unroll_safe def _flat_pycall_defaults(self, code, nargs, frame, defs_to_load): @@ -166,7 +166,7 @@ for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) new_frame.fastlocals_w[i] = w_arg - + defs_w = self.defs_w ndefs = len(defs_w) start = ndefs-defs_to_load @@ -174,7 +174,7 @@ for j in xrange(start, ndefs): new_frame.fastlocals_w[i] = defs_w[j] i += 1 - return new_frame.run() + return new_frame.run() def getdict(self): if self.w_func_dict is None: @@ -188,7 +188,7 @@ # unwrapping is done through unwrap_specs in typedef.py - def descr_function__new__(space, w_subtype, w_code, w_globals, + def descr_function__new__(space, w_subtype, w_code, w_globals, w_name=None, w_argdefs=None, w_closure=None): code = space.interp_w(Code, w_code) if not space.is_true(space.isinstance(w_globals, space.w_dict)): @@ -229,7 +229,7 @@ return self.getrepr(self.space, 'function %s' % (self.name,)) - # delicate + # delicate _all = {'': None} def _freeze_(self): @@ -260,7 +260,7 @@ new_inst = mod.get('builtin_function') return space.newtuple([new_inst, space.newtuple([space.wrap(code.identifier)])]) - + new_inst = mod.get('func_new') w = space.wrap if self.closure is None: @@ -524,7 +524,7 @@ space = self.space other = space.interpclass_w(w_other) if not isinstance(other, Method): - return space.w_False + return space.w_NotImplemented if self.w_instance is None: if other.w_instance is not None: return space.w_False @@ -562,7 +562,7 @@ else: tup = [self.w_function, w_instance, self.w_class] return space.newtuple([new_inst, space.newtuple(tup)]) - + class StaticMethod(Wrappable): """The staticmethod objects.""" _immutable_ = True Modified: pypy/trunk/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_function.py (original) +++ pypy/trunk/pypy/interpreter/test/test_function.py Thu Dec 9 19:27:50 2010 @@ -494,6 +494,18 @@ raises(TypeError, lambda: f(*0)) raises(TypeError, lambda: f(**0)) + def test_method_equal(self): + class A(object): + def m(self): + pass + + class X(object): + def __eq__(self, other): + return True + + assert A().m == X() + assert X() == A().m + class TestMethod: def setup_method(self, method): From afa at codespeak.net Thu Dec 9 20:48:45 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Thu, 9 Dec 2010 20:48:45 +0100 (CET) Subject: [pypy-svn] r79945 - in pypy/branch/fast-forward/pypy/translator/c: src test Message-ID: <20101209194845.28FB0282B9D@codespeak.net> Author: afa Date: Thu Dec 9 20:48:43 2010 New Revision: 79945 Modified: pypy/branch/fast-forward/pypy/translator/c/src/dtoa.c pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py Log: dtoa.c starts to give sensible results. Modified: pypy/branch/fast-forward/pypy/translator/c/src/dtoa.c ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/src/dtoa.c (original) +++ pypy/branch/fast-forward/pypy/translator/c/src/dtoa.c Thu Dec 9 20:48:43 2010 @@ -116,14 +116,17 @@ /* Begin PYPY hacks */ /* #include "Python.h" */ -#define IEEE_8087 +#define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #define HAVE_UINT32_T #define HAVE_INT32_T -#define PY_UINT32_T int +#define HAVE_UINT64_T +#define PY_UINT32_T unsigned int #define PY_INT32_T int +#define PY_UINT64_T unsigned long long #include #include #include +#include #define PyMem_Malloc PyObject_Malloc #define PyMem_Free PyObject_Free /* End PYPY hacks */ Modified: pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/test/test_dtoa.py Thu Dec 9 20:48:43 2010 @@ -51,31 +51,42 @@ raise ValueError("invalid input at position %d" % (offset,)) return result -def dtoa(value): - mode = 2 - precision = 3 +def dtoa(value, mode=0, precision=0): builder = StringBuilder(20) with lltype.scoped_alloc(rffi.INTP.TO, 1) as decpt_ptr: with lltype.scoped_alloc(rffi.INTP.TO, 1) as sign_ptr: with lltype.scoped_alloc(rffi.CCHARPP.TO, 1) as end_ptr: output_ptr = dg_dtoa(value, mode, precision, decpt_ptr, sign_ptr, end_ptr) - buflen = (rffi.cast(rffi.LONG, end_ptr[0]) - - rffi.cast(rffi.LONG, output_ptr)) - builder.append(rffi.charpsize2str(output_ptr, decpt_ptr[0])) - builder.append('.') - ptr = rffi.ptradd(output_ptr, - rffi.cast(lltype.Signed, decpt_ptr[0])) - buflen -= decpt_ptr[0] - builder.append(rffi.charpsize2str(ptr, buflen)) - dg_freedtoa(output_ptr) + try: + buflen = (rffi.cast(rffi.LONG, end_ptr[0]) - + rffi.cast(rffi.LONG, output_ptr)) + intpart = rffi.cast(lltype.Signed, decpt_ptr[0]) + if intpart <= buflen: + builder.append(rffi.charpsize2str(output_ptr, intpart)) + else: + builder.append(rffi.charpsize2str(output_ptr, buflen)) + while buflen < intpart: + builder.append('0') + intpart -= 1 + builder.append('.') + fracpart = buflen - intpart + if fracpart > 0: + ptr = rffi.ptradd(output_ptr, intpart) + builder.append(rffi.charpsize2str(ptr, fracpart)) + finally: + dg_freedtoa(output_ptr) return builder.build() def test_strtod(): assert strtod("12345") == 12345.0 assert strtod("1.1") == 1.1 + assert strtod("3.47") == 3.47 raises(ValueError, strtod, "123A") def test_dtoa(): assert dtoa(3.47) == "3.47" assert dtoa(1.1) == "1.1" + assert dtoa(12.3577) == "12.3577" + assert dtoa(10) == "10." + assert dtoa(1e100) == "1" + "0" * 100 + "." From danchr at codespeak.net Fri Dec 10 11:25:35 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Fri, 10 Dec 2010 11:25:35 +0100 (CET) Subject: [pypy-svn] r79948 - pypy/extradoc/planning/hg-migration Message-ID: <20101210102535.03AF75080F@codespeak.net> Author: danchr Date: Fri Dec 10 11:25:33 2010 New Revision: 79948 Modified: pypy/extradoc/planning/hg-migration/usermap.txt Log: Add self to authormap. Modified: pypy/extradoc/planning/hg-migration/usermap.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/usermap.txt (original) +++ pypy/extradoc/planning/hg-migration/usermap.txt Fri Dec 10 11:25:33 2010 @@ -137,3 +137,4 @@ lamby=Chris Lamb mcherm=Michael Chermside zooko=Zooko Wilcox-O Hearn +danchr=Dan Villiom Podlaski Christiansen \ No newline at end of file From david at codespeak.net Fri Dec 10 11:51:37 2010 From: david at codespeak.net (david at codespeak.net) Date: Fri, 10 Dec 2010 11:51:37 +0100 (CET) Subject: [pypy-svn] r79949 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . helper test Message-ID: <20101210105137.9D4C5282BAD@codespeak.net> Author: david Date: Fri Dec 10 11:51:35 2010 New Revision: 79949 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py pypy/branch/arm-backend/pypy/jit/backend/arm/runner.py pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Log: improve handling of calls by avoiding spilling arround calls. Fix some more regalloc issues and add typed subclasses of tempbox so they can be stored on the stack. Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Fri Dec 10 11:51:35 2010 @@ -3,7 +3,8 @@ from pypy.jit.backend.arm import registers as r from pypy.jit.backend.arm.arch import WORD, FUNC_ALIGN, PC_OFFSET from pypy.jit.backend.arm.codebuilder import ARMv7Builder, ARMv7InMemoryBuilder -from pypy.jit.backend.arm.regalloc import ARMRegisterManager, ARMFrameManager +from pypy.jit.backend.arm.regalloc import (ARMRegisterManager, ARMFrameManager, + TempInt, TempPtr) from pypy.jit.backend.llsupport.regalloc import compute_vars_longevity, TempBox from pypy.jit.metainterp.history import (Const, ConstInt, ConstPtr, BoxInt, BoxPtr, AbstractFailDescr, @@ -405,12 +406,14 @@ fcond=c.AL i = 0 while i < len(operations): + regalloc.position = i op = operations[i] opnum = op.getopnum() if self.can_merge_with_next_guard(op, i, operations): fcond = self.operations_with_guard[opnum](self, op, operations[i+1], regalloc, fcond) i += 1 + regalloc.position = i else: fcond = self.operations[opnum](self, op, regalloc, fcond) i += 1 @@ -469,9 +472,9 @@ loc = None if isinstance(thing, Const): if isinstance(thing, ConstInt): - box = BoxInt(thing.value) + box = TempInt() elif isinstance(thing, ConstPtr): - box = BoxPtr() + box = TempPtr() else: box = TempBox() loc = regalloc.force_allocate_reg(box, Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py Fri Dec 10 11:51:35 2010 @@ -40,8 +40,8 @@ boxes.append(box) l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) boxes.append(box) - res = regalloc.force_allocate_reg(op.result, boxes) regalloc.possibly_free_vars(boxes) + res = regalloc.force_allocate_reg(op.result, boxes) regalloc.possibly_free_var(op.result) if l1.is_imm(): @@ -91,8 +91,8 @@ else: l1, box = self._ensure_value_is_boxed(arg1, regalloc, forbidden_vars=boxes) boxes.append(box) - res = regalloc.force_allocate_reg(op.result) regalloc.possibly_free_vars(boxes) + res = regalloc.force_allocate_reg(op.result) regalloc.possibly_free_var(op.result) inv = c.get_opposite_of(condition) Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Fri Dec 10 11:51:35 2010 @@ -124,15 +124,16 @@ self.mc.SMULL(res.value, r.ip.value, reg1.value, reg2.value, cond=fcond) self.mc.CMP_rr(r.ip.value, res.value, shifttype=shift.ASR, imm=31, cond=fcond) - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) if guard.getopnum() == rop.GUARD_OVERFLOW: - return self._emit_guard(guard, regalloc, c.NE) - if guard.getopnum() == rop.GUARD_NO_OVERFLOW: - return self._emit_guard(guard, regalloc, c.EQ) + fcond = self._emit_guard(guard, regalloc, c.NE) + elif guard.getopnum() == rop.GUARD_NO_OVERFLOW: + fcond = self._emit_guard(guard, regalloc, c.EQ) else: assert 0 + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) + return fcond emit_op_int_floordiv = gen_emit_op_by_helper_call('DIV') emit_op_int_mod = gen_emit_op_by_helper_call('MOD') @@ -219,15 +220,17 @@ def emit_op_guard_true(self, op, regalloc, fcond): l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) - regalloc.possibly_free_var(box) self.mc.CMP_ri(l0.value, 0) - return self._emit_guard(op, regalloc, c.NE) + fcond = self._emit_guard(op, regalloc, c.NE) + regalloc.possibly_free_var(box) + return fcond def emit_op_guard_false(self, op, regalloc, fcond): l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) - regalloc.possibly_free_var(box) self.mc.CMP_ri(l0.value, 0) - return self._emit_guard(op, regalloc, c.EQ) + fcond = self._emit_guard(op, regalloc, c.EQ) + regalloc.possibly_free_var(box) + return fcond def emit_op_guard_value(self, op, regalloc, fcond): boxes = list(op.getarglist()) @@ -240,14 +243,15 @@ boxes.append(box) else: l1 = regalloc.make_sure_var_in_reg(a1) - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) if l1.is_imm(): self.mc.CMP_ri(l0.value, l1.getint()) else: self.mc.CMP_rr(l0.value, l1.value) - return self._emit_guard(op, regalloc, c.EQ) + fcond = self._emit_guard(op, regalloc, c.EQ) + regalloc.possibly_free_vars(boxes) + regalloc.possibly_free_var(op.result) + return fcond emit_op_guard_nonnull = emit_op_guard_true emit_op_guard_isnull = emit_op_guard_false @@ -331,26 +335,47 @@ self._gen_path_to_exit_path(op, op.getarglist(), regalloc, c.AL) return fcond - def emit_op_call(self, op, regalloc, fcond, save_all_regs=False): + def emit_op_call(self, op, regalloc, fcond, spill_all_regs=False): adr = rffi.cast(lltype.Signed, op.getarg(0).getint()) args = op.getarglist()[1:] - cond = self._emit_call(adr, args, regalloc, fcond, save_all_regs, op.result) + cond = self._emit_call(adr, args, regalloc, fcond, op.result, spill_all_regs=spill_all_regs) + regalloc.possibly_free_vars(args) if op.result: regalloc.possibly_free_var(op.result) descr = op.getdescr() #XXX Hack, Hack, Hack if op.result and not we_are_translated() and not isinstance(descr, LoopToken): - loc = regalloc.loc(op.result) + loc = regalloc.call_result_location(op.result) size = descr.get_result_size(False) signed = descr.is_result_signed() self._ensure_result_bit_extension(loc, size, signed, regalloc) return cond - def _emit_call(self, adr, args, regalloc, fcond=c.AL, save_all_regs=False, result=None): - # all arguments past the 4th go on the stack + # XXX improve this interface + # XXX and get rid of spill_all_regs in favor of pushing them in + # emit_op_call_may_force + # XXX improve freeing of stuff here + def _emit_call(self, adr, args, regalloc, fcond=c.AL, result=None, spill_all_regs=False): n = 0 n_args = len(args) + reg_args = min(n_args, 4) + # prepare arguments passed in registers + for i in range(0, reg_args): + l = regalloc.make_sure_var_in_reg(args[i], + selected_reg=r.all_regs[i]) + # save caller saved registers + if spill_all_regs: + regalloc.before_call(save_all_regs=spill_all_regs) + else: + if result: + if reg_args > 0 and regalloc.stays_alive(args[0]): + regalloc.force_spill_var(args[0]) + self.mc.PUSH([reg.value for reg in r.caller_resp][1:]) + else: + self.mc.PUSH([reg.value for reg in r.caller_resp]) + + # all arguments past the 4th go on the stack if n_args > 4: stack_args = n_args - 4 n = stack_args*WORD @@ -360,23 +385,24 @@ self.mc.STR_ri(reg.value, r.sp.value, (i-4)*WORD) regalloc.possibly_free_var(box) - - reg_args = min(n_args, 4) - for i in range(0, reg_args): - l = regalloc.make_sure_var_in_reg(args[i], - selected_reg=r.all_regs[i]) - # XXX use PUSH here instead of spilling every reg for itself - regalloc.before_call(save_all_regs=save_all_regs) - regalloc.possibly_free_vars(args) - + #the actual call self.mc.BL(adr) - if result: - regalloc.after_call(result) + regalloc.possibly_free_vars(args) # readjust the sp in case we passed some args on the stack if n_args > 4: assert n > 0 self._adjust_sp(-n, regalloc, fcond=fcond) + + # restore the argumets stored on the stack + if spill_all_regs: + regalloc.after_call(result) + else: + if result is not None: + regalloc.after_call(result) + self.mc.POP([reg.value for reg in r.caller_resp][1:]) + else: + self.mc.POP([reg.value for reg in r.caller_resp]) return fcond def emit_op_same_as(self, op, regalloc, fcond): @@ -397,41 +423,41 @@ return fcond def emit_op_guard_no_exception(self, op, regalloc, fcond): - t = TempBox() - loc = regalloc.force_allocate_reg(t) - self.mc.gen_load_int(loc.value, self.cpu.pos_exception(), fcond) + loc, box = self._ensure_value_is_boxed( + ConstInt(self.cpu.pos_exception()), regalloc) + self.mc.LDR_ri(loc.value, loc.value) self.mc.CMP_ri(loc.value, 0) cond = self._emit_guard(op, regalloc, c.EQ, save_exc=True) - regalloc.possibly_free_var(t) + regalloc.possibly_free_var(box) return cond def emit_op_guard_exception(self, op, regalloc, fcond): - args = op.getarglist() - t = TempBox() - t1 = TempBox() - loc = regalloc.force_allocate_reg(t, args) - loc1 = regalloc.force_allocate_reg(t1, args + [t]) - self.mc.gen_load_int(loc.value, - rffi.cast(lltype.Signed, op.getarg(0).getint()), - fcond) + boxes = list(op.getarglist()) + arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) + loc, box = self._ensure_value_is_boxed(arg0, regalloc) + boxes.append(box) + loc1, box = self._ensure_value_is_boxed( + ConstInt(self.cpu.pos_exception()), regalloc, boxes) + boxes.append(box) + if op.result in regalloc.longevity: + resloc = regalloc.force_allocate_reg(op.result, boxes) + boxes.append(resloc) + else: + resloc = None - self.mc.gen_load_int(loc1.value, self.cpu.pos_exception(), fcond) self.mc.LDR_ri(loc1.value, loc1.value) self.mc.CMP_rr(loc1.value, loc.value) self._emit_guard(op, regalloc, c.EQ, save_exc=True) + regalloc.possibly_free_vars(boxes) self.mc.gen_load_int(loc1.value, self.cpu.pos_exc_value(), fcond) - if op.result in regalloc.longevity: - resloc = regalloc.force_allocate_reg(op.result, args + [t, t1]) + if resloc: self.mc.LDR_ri(resloc.value, loc1.value) - regalloc.possibly_free_var(op.result) self.mc.gen_load_int(loc.value, self.cpu.pos_exception(), fcond) self.mc.MOV_ri(r.ip.value, 0) self.mc.STR_ri(r.ip.value, loc.value) self.mc.STR_ri(r.ip.value, loc1.value) - regalloc.possibly_free_var(t) - regalloc.possibly_free_var(t1) return fcond def emit_op_debug_merge_point(self, op, regalloc, fcond): @@ -470,9 +496,9 @@ ofs, size, ptr = self._unpack_fielddescr(op.getdescr()) # ofs_loc = regalloc.make_sure_var_in_reg(ConstInt(ofs)) base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc) - res = regalloc.force_allocate_reg(op.result, [a0]) regalloc.possibly_free_var(a0) regalloc.possibly_free_var(base_box) + res = regalloc.force_allocate_reg(op.result, [a0]) regalloc.possibly_free_var(op.result) if size == 4: @@ -933,9 +959,8 @@ jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(asm_helper_adr, [t, op.getarg(0)], regalloc, fcond, False, op.result) + self._emit_call(asm_helper_adr, [t, op.getarg(0)], regalloc, fcond, op.result) regalloc.possibly_free_var(t) - # jump to merge point jmp_pos = self.mc.currpos() jmp_location = self.mc.curraddr() @@ -985,12 +1010,12 @@ l0 = regalloc.force_allocate_reg(t) self.mc.LDR_ri(l0.value, r.fp.value) self.mc.CMP_ri(l0.value, 0) + + self._emit_guard(guard_op, regalloc, c.GE) regalloc.possibly_free_var(t) regalloc.possibly_free_vars_for_op(op) if op.result: regalloc.possibly_free_var(op.result) - - self._emit_guard(guard_op, regalloc, c.GE) return fcond def emit_guard_call_may_force(self, op, guard_op, regalloc, fcond): @@ -999,15 +1024,15 @@ self._write_fail_index(fail_index, regalloc) # force all reg values to be spilled when calling - fcond = self.emit_op_call(op, regalloc, fcond, save_all_regs=True) + fcond = self.emit_op_call(op, regalloc, fcond, spill_all_regs=True) t = TempBox() l0 = regalloc.force_allocate_reg(t) self.mc.LDR_ri(l0.value, r.fp.value) self.mc.CMP_ri(l0.value, 0) - regalloc.possibly_free_var(t) self._emit_guard(guard_op, regalloc, c.GE) + regalloc.possibly_free_var(t) return fcond def _write_fail_index(self, fail_index, regalloc): @@ -1046,19 +1071,23 @@ vloc, v = self._ensure_value_is_boxed(v, regalloc, [res_v]) boxes.append(v) size, size_box = self._ensure_value_is_boxed(itemsize_box, regalloc, boxes) + boxes.append(size_box) self.mc.MUL(size.value, size.value, vloc.value) if ofs_items_loc.is_imm(): self.mc.ADD_ri(size.value, size.value, ofs_items_loc.value) else: self.mc.ADD_rr(size.value, size.value, ofs_items_loc.value) - self._emit_call(self.malloc_func_addr, [size_box], regalloc, result=res_v) + self._emit_call(self.malloc_func_addr, [size_box], regalloc, + result=res_v) base_loc = regalloc.make_sure_var_in_reg(res_v) value_loc = regalloc.make_sure_var_in_reg(v) + regalloc.possibly_free_vars(boxes) + assert value_loc.is_reg() + assert base_loc.is_reg() self.mc.STR_ri(value_loc.value, base_loc.value, ofs_length) - regalloc.possibly_free_vars(boxes) def emit_op_new(self, op, regalloc, fcond): arglocs = self._prepare_args_for_new_op(op.getdescr(), regalloc) @@ -1074,18 +1103,19 @@ arglocs = self._prepare_args_for_new_op(descrsize, regalloc) self._emit_call(self.malloc_func_addr, arglocs, regalloc, result=op.result) - resloc = regalloc.loc(op.result) # r0 - self.set_vtable(resloc, classint, regalloc) + self.set_vtable(op.result, classint, regalloc) regalloc.possibly_free_vars(arglocs) regalloc.possibly_free_var(op.result) return fcond - def set_vtable(self, loc, vtable, regalloc): + def set_vtable(self, box, vtable, regalloc): if self.cpu.vtable_offset is not None: - assert loc.is_reg() + loc = regalloc.loc(box) # r0 + assert loc is r.r0 adr = rffi.cast(lltype.Signed, vtable) t = TempBox() - loc_vtable = regalloc.force_allocate_reg(t) + loc_vtable = regalloc.force_allocate_reg(t, [box]) + assert loc_vtable is not loc regalloc.possibly_free_var(t) self.mc.gen_load_int(loc_vtable.value, adr) self.mc.STR_ri(loc_vtable.value, loc.value, self.cpu.vtable_offset) Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Fri Dec 10 11:51:35 2010 @@ -2,9 +2,19 @@ RegisterManager, compute_vars_longevity, TempBox from pypy.jit.backend.arm import registers as r from pypy.jit.backend.arm import locations -from pypy.jit.metainterp.history import ConstInt, ConstPtr +from pypy.jit.metainterp.history import ConstInt, ConstPtr, REF, INT from pypy.rpython.lltypesystem import rffi, lltype +class TempInt(TempBox): + type = INT + + def __repr__(self): + return "" % (id(self),) +class TempPtr(TempBox): + type = REF + def __repr__(self): + return "" % (id(self),) + class ARMRegisterManager(RegisterManager): all_regs = r.all_regs box_types = None # or a list of acceptable types @@ -46,6 +56,7 @@ # is also used on op args, which is a non-resizable list self.possibly_free_vars(list(inputargs)) + #XXX remove def force_allocate_reg(self, v, forbidden_vars=[], selected_reg=None, need_lower_byte=False): # override ../llsupport/regalloc.py to set longevity for vals not in longevity @@ -64,6 +75,10 @@ self.reg_bindings[v] = loc return loc + def force_spill_var(self, var): + self._sync_var(var) + del self.reg_bindings[var] + class ARMFrameManager(FrameManager): def __init__(self): FrameManager.__init__(self) Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/runner.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/runner.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/runner.py Fri Dec 10 11:51:35 2010 @@ -78,13 +78,14 @@ faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = -1 # start of "no gc operation!" block + frame_depth = faildescr._arm_frame_depth addr_end_of_frame = (addr_of_force_index - - (faildescr._arm_frame_depth+len(all_regs))*WORD) + (frame_depth+len(all_regs))*WORD) fail_index_2 = self.assembler.failure_recovery_func( faildescr._failure_recovery_code, addr_of_force_index, addr_end_of_frame) self.assembler.leave_jitted_hook() # end of "no gc operation!" block - #assert fail_index == fail_index_2 + assert fail_index == fail_index_2 return faildescr Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/test/test_zll_random.py Fri Dec 10 11:51:35 2010 @@ -9,6 +9,6 @@ def test_stress(): cpu = CPU(None, None) - r = Random() for i in range(1000): + r = Random(i) check_random_function(cpu, LLtypeOperationBuilder, r, i, 1000) From danchr at codespeak.net Fri Dec 10 12:16:09 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Fri, 10 Dec 2010 12:16:09 +0100 (CET) Subject: [pypy-svn] r79950 - pypy/extradoc/planning/hg-migration Message-ID: <20101210111609.4EB8F5080F@codespeak.net> Author: danchr Date: Fri Dec 10 12:16:07 2010 New Revision: 79950 Modified: pypy/extradoc/planning/hg-migration/usermap.txt Log: Add missing newline at end of file. Modified: pypy/extradoc/planning/hg-migration/usermap.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/usermap.txt (original) +++ pypy/extradoc/planning/hg-migration/usermap.txt Fri Dec 10 12:16:07 2010 @@ -137,4 +137,4 @@ lamby=Chris Lamb mcherm=Michael Chermside zooko=Zooko Wilcox-O Hearn -danchr=Dan Villiom Podlaski Christiansen \ No newline at end of file +danchr=Dan Villiom Podlaski Christiansen From danchr at codespeak.net Fri Dec 10 12:17:06 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Fri, 10 Dec 2010 12:17:06 +0100 (CET) Subject: [pypy-svn] r79951 - pypy/extradoc/planning/hg-migration Message-ID: <20101210111706.B084A282BAD@codespeak.net> Author: danchr Date: Fri Dec 10 12:17:05 2010 New Revision: 79951 Modified: pypy/extradoc/planning/hg-migration/usermap.txt Log: Add a few Python personas. Modified: pypy/extradoc/planning/hg-migration/usermap.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/usermap.txt (original) +++ pypy/extradoc/planning/hg-migration/usermap.txt Fri Dec 10 12:17:05 2010 @@ -50,7 +50,7 @@ exarkun=Jean-Paul Calderone bgola=Bruno Gola dan=Daniel Roberts -bob=Bob Ippolito +bob=Bob Ippolito afayolle=Alexandre Fayolle simonb=Simon Burton alastair=alastair @@ -60,13 +60,13 @@ dialtone=Valentino Volonghi magcius=Jean-Philippe St. Pierre trundle=Andreas St?hrk -gvanrossum=Guido van Rossum +gvanrossum=Guido van Rossum vinogradov=Pavel Vinogradov jum=Jens-Uwe Mager wlav=Wim Lavrijsen akuhn=Adrian Kuhn pdg=Paul deGrandis -gbrandl=Georg Brandl +gbrandl=Georg Brandl gromit=Gerald Klix wanja=Wanja Saatkamp boria=boria @@ -95,7 +95,7 @@ karlb=Karl Bartel odie=Olivier Dormond haypo=Victor Stinner -antoine=Antoine Pitrou +antoine=Antoine Pitrou atobe=Toby Watson micke=Mikael Sch?nenberg nshepperd=Neil Shepperd @@ -128,7 +128,7 @@ jbaker=Jim Baker jgilbert=Joshua Gilbert misto=Fabrizio Milo -niemeyer=Gustavo Niemeyer +niemeyer=Gustavo Niemeyer radix=Christopher Armstrong verte=William Leslie yusei=Yusei Tahara From danchr at codespeak.net Fri Dec 10 12:26:09 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Fri, 10 Dec 2010 12:26:09 +0100 (CET) Subject: [pypy-svn] r79952 - pypy/extradoc/planning/hg-migration Message-ID: <20101210112609.815FE282BAD@codespeak.net> Author: danchr Date: Fri Dec 10 12:26:08 2010 New Revision: 79952 Modified: pypy/extradoc/planning/hg-migration/usermap.txt Log: Correct a mail to be the same used in the pymigr map for CPython. Modified: pypy/extradoc/planning/hg-migration/usermap.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/usermap.txt (original) +++ pypy/extradoc/planning/hg-migration/usermap.txt Fri Dec 10 12:26:08 2010 @@ -128,7 +128,7 @@ jbaker=Jim Baker jgilbert=Joshua Gilbert misto=Fabrizio Milo -niemeyer=Gustavo Niemeyer +niemeyer=Gustavo Niemeyer radix=Christopher Armstrong verte=William Leslie yusei=Yusei Tahara From danchr at codespeak.net Fri Dec 10 13:35:32 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Fri, 10 Dec 2010 13:35:32 +0100 (CET) Subject: [pypy-svn] r79953 - pypy/extradoc/planning/hg-migration Message-ID: <20101210123532.7E621282BAD@codespeak.net> Author: danchr Date: Fri Dec 10 13:35:23 2010 New Revision: 79953 Modified: pypy/extradoc/planning/hg-migration/usermap.txt Log: Add codespeak.net for all entries that lack a mail address. These addresses are already public, as they are used for Subversion commit notifications. Modified: pypy/extradoc/planning/hg-migration/usermap.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/usermap.txt (original) +++ pypy/extradoc/planning/hg-migration/usermap.txt Fri Dec 10 13:35:23 2010 @@ -20,121 +20,121 @@ xoraxax=Alexander Schremmer rxe=Richard Emslie ale=Anders Lehmann -auc=Aurelien Campeas -getxsick=Bartosz Skowron -nik=Niklaus Haldimann -cami=Camillo Bruni +auc=Aurelien Campeas +getxsick=Bartosz Skowron +nik=Niklaus Haldimann +cami=Camillo Bruni lac=Laura Creighton david=David Schneider -sanxiyn=Seo Sanghyeon +sanxiyn=Seo Sanghyeon santagada=Leonardo Santagada -tverwaes=Toon Verwaest -adim=Adrien Di Mascio -rhymes=Lawrence Oluyede +tverwaes=Toon Verwaest +adim=Adrien Di Mascio +rhymes=Lawrence Oluyede jacob=Jacob Hallen -guido=Guido Wesdorp -ludal=Ludovic Aubry +guido=Guido Wesdorp +ludal=Ludovic Aubry jlg=Jakub Gustak bea=Beatrice During hakanardo=Hakan Ardo niko=Niko Matsakis -alex=Alex Martelli -jcreigh=Jason Creighton +alex=Alex Martelli +jcreigh=Jason Creighton iko=Anders Hammarquist agaynor=Alex Gaynor -stephan=Stephan Diehl -jandem=Jan de Mooij -pmaupin=Patrick Maupin -sschwarzer=Stefan Schwarzer -tomek=Tomek Meka +stephan=Stephan Diehl +jandem=Jan de Mooij +pmaupin=Patrick Maupin +sschwarzer=Stefan Schwarzer +tomek=Tomek Meka exarkun=Jean-Paul Calderone -bgola=Bruno Gola +bgola=Bruno Gola dan=Daniel Roberts bob=Bob Ippolito -afayolle=Alexandre Fayolle -simonb=Simon Burton +afayolle=Alexandre Fayolle +simonb=Simon Burton alastair=alastair mgedmin=Marius Gedminas -witulski=John Witulski -nico=Nicolas Chauvat -dialtone=Valentino Volonghi -magcius=Jean-Philippe St. Pierre +witulski=John Witulski +nico=Nicolas Chauvat +dialtone=Valentino Volonghi +magcius=Jean-Philippe St. Pierre trundle=Andreas St?hrk gvanrossum=Guido van Rossum -vinogradov=Pavel Vinogradov -jum=Jens-Uwe Mager -wlav=Wim Lavrijsen -akuhn=Adrian Kuhn -pdg=Paul deGrandis +vinogradov=Pavel Vinogradov +jum=Jens-Uwe Mager +wlav=Wim Lavrijsen +akuhn=Adrian Kuhn +pdg=Paul deGrandis gbrandl=Georg Brandl -gromit=Gerald Klix -wanja=Wanja Saatkamp +gromit=Gerald Klix +wanja=Wanja Saatkamp boria=boria -davide=Davide Ancona -oscar=Oscar Nierstrasz -goden=Eugene Oden -leuschel=Michael Leuschel -docgok=Henry Mason -guenter=Guenter Jantzen -bert=Bert Freudenberg -lukas=Lukas Renggli -lene=Lene Wagner -regmee=Amit Regmi -adurdin=Andrew Durdin -benyoung=Ben Young -bigdog=Michael Schneider -briandorsey=Brian Dorsey -njriley=Nicholas Riley -igorto=Igor Trindade Oliveira -micktwomey=Michael Twomey -rocco=Rocco Moretti +davide=Davide Ancona +oscar=Oscar Nierstrasz +goden=Eugene Oden +leuschel=Michael Leuschel +docgok=Henry Mason +guenter=Guenter Jantzen +bert=Bert Freudenberg +lukas=Lukas Renggli +lene=Lene Wagner +regmee=Amit Regmi +adurdin=Andrew Durdin +benyoung=Ben Young +bigdog=Michael Schneider +briandorsey=Brian Dorsey +njriley=Nicholas Riley +igorto=Igor Trindade Oliveira +micktwomey=Michael Twomey +rocco=Rocco Moretti wildchild=Gabriel Lavoie -lucian=Lucian Branescu Mihaila -dinu=Dinu Gherman -jared.grubb=Jared Grubb -karlb=Karl Bartel +lucian=Lucian Branescu Mihaila +dinu=Dinu Gherman +jared.grubb=Jared Grubb +karlb=Karl Bartel odie=Olivier Dormond -haypo=Victor Stinner +haypo=Victor Stinner antoine=Antoine Pitrou -atobe=Toby Watson +atobe=Toby Watson micke=Mikael Sch?nenberg -nshepperd=Neil Shepperd -stuart=Stuart Williams -hruske=Gasper Zejn -justas=Justas Sadzevicius -syt=Sylvain Thenault -alecu=Alejandro J. Cura -electronicru=Alexander Sedov -elmom=Elmo M?ntynen -jriehl=Jonathan David Riehl -quest=Anders Qvist -amcintyre=Alan McIntyre -tobami=Miquel Torres -alix=Alix Einfeldt -pzieschang=Pieter Zieschang -aft=Andrew Thompson -blais=Martin Blais -busemann=Stephan Busemann -esmljaos=Jacob Oscarson -henrikv=Henrik Vendelbo -iammisc=Travis Francis Athougies -laszlo=Artur Lisiecki -lucio=Lucio Torre -lutz_p=Lutz Paelike -tav=tav -asigfrid=Anders Sigfridsson -gotcha=Godefroid Chappelle -jacek=Jacek Generowicz -jbaker=Jim Baker -jgilbert=Joshua Gilbert -misto=Fabrizio Milo +nshepperd=Neil Shepperd +stuart=Stuart Williams +hruske=Gasper Zejn +justas=Justas Sadzevicius +syt=Sylvain Thenault +alecu=Alejandro J. Cura +electronicru=Alexander Sedov +elmom=Elmo M?ntynen +jriehl=Jonathan David Riehl +quest=Anders Qvist +amcintyre=Alan McIntyre +tobami=Miquel Torres +alix=Alix Einfeldt +pzieschang=Pieter Zieschang +aft=Andrew Thompson +blais=Martin Blais +busemann=Stephan Busemann +esmljaos=Jacob Oscarson +henrikv=Henrik Vendelbo +iammisc=Travis Francis Athougies +laszlo=Artur Lisiecki +lucio=Lucio Torre +lutz_p=Lutz Paelike +tav=tav at codespeak.net +asigfrid=Anders Sigfridsson +gotcha=Godefroid Chappelle +jacek=Jacek Generowicz +jbaker=Jim Baker +jgilbert=Joshua Gilbert +misto=Fabrizio Milo niemeyer=Gustavo Niemeyer -radix=Christopher Armstrong +radix=Christopher Armstrong verte=William Leslie -yusei=Yusei Tahara -anthon=Anthon van der Neut -jan=Jan Balster -lamby=Chris Lamb -mcherm=Michael Chermside -zooko=Zooko Wilcox-O Hearn +yusei=Yusei Tahara +anthon=Anthon van der Neut +jan=Jan Balster +lamby=Chris Lamb +mcherm=Michael Chermside +zooko=Zooko Wilcox-O Hearn danchr=Dan Villiom Podlaski Christiansen From danchr at codespeak.net Fri Dec 10 13:41:51 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Fri, 10 Dec 2010 13:41:51 +0100 (CET) Subject: [pypy-svn] r79954 - pypy/extradoc/planning/hg-migration Message-ID: <20101210124151.C130350810@codespeak.net> Author: danchr Date: Fri Dec 10 13:41:50 2010 New Revision: 79954 Modified: pypy/extradoc/planning/hg-migration/usermap.txt Log: Add missing authors. Modified: pypy/extradoc/planning/hg-migration/usermap.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/usermap.txt (original) +++ pypy/extradoc/planning/hg-migration/usermap.txt Fri Dec 10 13:41:50 2010 @@ -138,3 +138,8 @@ mcherm=Michael Chermside zooko=Zooko Wilcox-O Hearn danchr=Dan Villiom Podlaski Christiansen +anna=Anna Ravencroft +dcolish=Dan Colish +dstromberg=Dan Stromberg +gintas=Gintautas Miliauskas +ignas=Ignas Mikalajunas From danchr at codespeak.net Fri Dec 10 13:46:29 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Fri, 10 Dec 2010 13:46:29 +0100 (CET) Subject: [pypy-svn] r79955 - pypy/extradoc/planning/hg-migration Message-ID: <20101210124629.9436F282BAD@codespeak.net> Author: danchr Date: Fri Dec 10 13:46:27 2010 New Revision: 79955 Modified: pypy/extradoc/planning/hg-migration/usermap.txt Log: Update tav as per his request. Modified: pypy/extradoc/planning/hg-migration/usermap.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/usermap.txt (original) +++ pypy/extradoc/planning/hg-migration/usermap.txt Fri Dec 10 13:46:27 2010 @@ -121,7 +121,7 @@ laszlo=Artur Lisiecki lucio=Lucio Torre lutz_p=Lutz Paelike -tav=tav at codespeak.net +tav=tav asigfrid=Anders Sigfridsson gotcha=Godefroid Chappelle jacek=Jacek Generowicz From danchr at codespeak.net Fri Dec 10 15:00:32 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Fri, 10 Dec 2010 15:00:32 +0100 (CET) Subject: [pypy-svn] r79956 - pypy/extradoc/planning/hg-migration Message-ID: <20101210140032.87C97282B9E@codespeak.net> Author: danchr Date: Fri Dec 10 15:00:29 2010 New Revision: 79956 Modified: pypy/extradoc/planning/hg-migration/usermap.txt Log: Add the name for 'boria'. There is no 'boria' user on codespeak, so no email is added. However, the username is retained. Modified: pypy/extradoc/planning/hg-migration/usermap.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/usermap.txt (original) +++ pypy/extradoc/planning/hg-migration/usermap.txt Fri Dec 10 15:00:29 2010 @@ -69,7 +69,7 @@ gbrandl=Georg Brandl gromit=Gerald Klix wanja=Wanja Saatkamp -boria=boria +boria=Boris Feigin davide=Davide Ancona oscar=Oscar Nierstrasz goden=Eugene Oden From arigo at codespeak.net Fri Dec 10 15:40:13 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Dec 2010 15:40:13 +0100 (CET) Subject: [pypy-svn] r79957 - pypy/trunk/pypy/rpython/memory/gc Message-ID: <20101210144013.69553282B9E@codespeak.net> Author: arigo Date: Fri Dec 10 15:40:11 2010 New Revision: 79957 Modified: pypy/trunk/pypy/rpython/memory/gc/minimark.py Log: Also print the old nursery address. Modified: pypy/trunk/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/minimark.py Fri Dec 10 15:40:11 2010 @@ -423,7 +423,8 @@ llarena.arena_protect(newnurs, self._nursery_memory_size(), False) self.nursery = newnurs self.nursery_top = self.nursery + self.nursery_size - debug_print("switching to nursery", self.nursery, + debug_print("switching from nursery", oldnurs, + "to nursery", self.nursery, "size", self.nursery_size) debug_stop("gc-debug") From afa at codespeak.net Fri Dec 10 16:01:49 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Fri, 10 Dec 2010 16:01:49 +0100 (CET) Subject: [pypy-svn] r79958 - in pypy/trunk/pypy: rlib rpython/lltypesystem Message-ID: <20101210150149.EA20B282B9E@codespeak.net> Author: afa Date: Fri Dec 10 16:01:48 2010 New Revision: 79958 Modified: pypy/trunk/pypy/rlib/rmmap.py pypy/trunk/pypy/rpython/lltypesystem/llarena.py Log: Don't release the GIL when VirtualProtect is called to protect recently freed memory. Modified: pypy/trunk/pypy/rlib/rmmap.py ============================================================================== --- pypy/trunk/pypy/rlib/rmmap.py (original) +++ pypy/trunk/pypy/rlib/rmmap.py Fri Dec 10 16:01:48 2010 @@ -100,8 +100,11 @@ sandboxsafe=True, threadsafe=False) return unsafe, safe -def winexternal(name, args, result): - return rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_, calling_conv='win') +def winexternal(name, args, result, **kwargs): + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_, + calling_conv='win', + **kwargs) PTR = rffi.CCHARP @@ -189,9 +192,17 @@ VirtualAlloc = winexternal('VirtualAlloc', [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD], rffi.VOIDP) - VirtualProtect = winexternal('VirtualProtect', - [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], - BOOL) + # VirtualProtect is used in llarena and should not release the GIL + _VirtualProtect = winexternal('VirtualProtect', + [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], + BOOL, + _nowrapper=True) + def VirtualProtect(addr, size, mode, oldmode_ptr): + return _VirtualProtect(addr, + rffi.cast(rffi.SIZE_T, size), + rffi.cast(DWORD, mode), + oldmode_ptr) + VirtualProtect._annspecialcase_ = 'specialize:ll' VirtualFree = winexternal('VirtualFree', [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL) Modified: pypy/trunk/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/llarena.py Fri Dec 10 16:01:48 2010 @@ -531,9 +531,7 @@ from pypy.rlib.rmmap import PAGE_READWRITE as newprotect arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') VirtualProtect(rffi.cast(rffi.VOIDP, addr), - rffi.cast(rffi.SIZE_T, size), - newprotect, - arg) + size, newprotect, arg) # ignore potential errors lltype.free(arg, flavor='raw') has_protect = True From arigo at codespeak.net Fri Dec 10 16:20:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Dec 2010 16:20:32 +0100 (CET) Subject: [pypy-svn] r79959 - pypy/trunk/pypy/rpython/memory/gc Message-ID: <20101210152032.696CA282B9E@codespeak.net> Author: arigo Date: Fri Dec 10 16:20:31 2010 New Revision: 79959 Modified: pypy/trunk/pypy/rpython/memory/gc/env.py Log: Bah, messed up: on Mac OS X, it would return exactly the L2 cache size as a "best nursery size". That's really wrong. That's probably the worst possible answer :-( Modified: pypy/trunk/pypy/rpython/memory/gc/env.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/env.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/env.py Fri Dec 10 16:20:31 2010 @@ -4,7 +4,7 @@ import os, sys from pypy.rlib.rarithmetic import r_uint from pypy.rlib.debug import debug_print, debug_start, debug_stop -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.lltypesystem.lloperation import llop # ____________________________________________________________ @@ -110,13 +110,7 @@ # ____________________________________________________________ # Estimation of the nursery size, based on the L2 cache. -def best_nursery_size_for_L2cache(L2cache): - # Heuristically, the best nursery size to choose is about half - # of the L2 cache. XXX benchmark some more. - if L2cache > 0: - return L2cache // 2 - else: - return -1 +# ---------- Linux2 ---------- def get_L2cache_linux2(filename): debug_start("gc-hardware") @@ -189,61 +183,66 @@ pos += 1 return pos +# ---------- Darwin ---------- -if sys.platform == 'linux2': - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. Linux code.""" - L2cache = get_L2cache_linux2('/proc/cpuinfo') - return best_nursery_size_for_L2cache(L2cache) - -elif sys.platform == 'darwin': - from pypy.rpython.lltypesystem import rffi - - sysctlbyname = rffi.llexternal('sysctlbyname', - [rffi.CCHARP, rffi.VOIDP, rffi.SIZE_TP, - rffi.VOIDP, rffi.SIZE_T], - rffi.INT, - sandboxsafe=True) - - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. - """ - debug_start("gc-hardware") - L2cache = 0 - l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') +sysctlbyname = rffi.llexternal('sysctlbyname', + [rffi.CCHARP, rffi.VOIDP, rffi.SIZE_TP, + rffi.VOIDP, rffi.SIZE_T], + rffi.INT, + sandboxsafe=True) +def get_L2cache_darwin(): + """Try to estimate the best nursery size at run-time, depending + on the machine we are running on. + """ + debug_start("gc-hardware") + L2cache = 0 + l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') + try: + len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') try: - len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') - try: - size = rffi.sizeof(rffi.LONGLONG) - l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) - len_p[0] = rffi.cast(rffi.SIZE_T, size) - # XXX a hack for llhelper not being robust-enough - result = sysctlbyname("hw.l2cachesize", - rffi.cast(rffi.VOIDP, l2cache_p), - len_p, - lltype.nullptr(rffi.VOIDP.TO), - rffi.cast(rffi.SIZE_T, 0)) - if (rffi.cast(lltype.Signed, result) == 0 and - rffi.cast(lltype.Signed, len_p[0]) == size): - L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) - if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: - L2cache = 0 # overflow! - finally: - lltype.free(len_p, flavor='raw') + size = rffi.sizeof(rffi.LONGLONG) + l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) + len_p[0] = rffi.cast(rffi.SIZE_T, size) + # XXX a hack for llhelper not being robust-enough + result = sysctlbyname("hw.l2cachesize", + rffi.cast(rffi.VOIDP, l2cache_p), + len_p, + lltype.nullptr(rffi.VOIDP.TO), + rffi.cast(rffi.SIZE_T, 0)) + if (rffi.cast(lltype.Signed, result) == 0 and + rffi.cast(lltype.Signed, len_p[0]) == size): + L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) + if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: + L2cache = 0 # overflow! finally: - lltype.free(l2cache_p, flavor='raw') - debug_print("L2cache =", L2cache) - debug_stop("gc-hardware") - if L2cache > 0: - return L2cache - else: - # Print a top-level warning even in non-debug builds - llop.debug_print(lltype.Void, - "Warning: cannot find your CPU L2 cache size with sysctl()") - return -1 + lltype.free(len_p, flavor='raw') + finally: + lltype.free(l2cache_p, flavor='raw') + debug_print("L2cache =", L2cache) + debug_stop("gc-hardware") + if L2cache > 0: + return L2cache + else: + # Print a top-level warning even in non-debug builds + llop.debug_print(lltype.Void, + "Warning: cannot find your CPU L2 cache size with sysctl()") + return -1 -else: - def estimate_best_nursery_size(): - return -1 # XXX implement me for other platforms +# -------------------- + +get_L2cache = globals().get('get_L2cache_' + sys.platform, + lambda: -1) # implement me for other platforms + +def best_nursery_size_for_L2cache(L2cache): + # Heuristically, the best nursery size to choose is about half + # of the L2 cache. + if L2cache > 0: + return L2cache // 2 + else: + return -1 + +def estimate_best_nursery_size(): + """Try to estimate the best nursery size at run-time, depending + on the machine we are running on. Linux code.""" + L2cache = get_L2cache() + return best_nursery_size_for_L2cache(L2cache) From arigo at codespeak.net Fri Dec 10 17:00:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Dec 2010 17:00:47 +0100 (CET) Subject: [pypy-svn] r79960 - pypy/extradoc/sprintinfo/leysin-winter-2011 Message-ID: <20101210160047.5CD83282BAD@codespeak.net> Author: arigo Date: Fri Dec 10 17:00:45 2010 New Revision: 79960 Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Log: Add Michael Foord :-) Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Fri Dec 10 17:00:45 2010 @@ -12,6 +12,7 @@ ==================== ============== ======================= Armin Rigo --/23 private Antonio Cuni 15/22 ermina +Michael Foord 16/22 (maybe) ermina ==================== ============== ======================= From arigo at codespeak.net Fri Dec 10 18:00:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Dec 2010 18:00:24 +0100 (CET) Subject: [pypy-svn] r79961 - pypy/branch/ctypes-fix Message-ID: <20101210170024.F045750810@codespeak.net> Author: arigo Date: Fri Dec 10 18:00:23 2010 New Revision: 79961 Added: pypy/branch/ctypes-fix/ - copied from r79960, pypy/trunk/ Log: A branch to check in a (hopefully) quick fix of ctypes's c_void_p. From arigo at codespeak.net Fri Dec 10 18:03:04 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Fri, 10 Dec 2010 18:03:04 +0100 (CET) Subject: [pypy-svn] r79962 - in pypy/branch/ctypes-fix: lib_pypy/_ctypes pypy/module/test_lib_pypy/ctypes_tests Message-ID: <20101210170304.08A82282BAD@codespeak.net> Author: arigo Date: Fri Dec 10 18:03:02 2010 New Revision: 79962 Modified: pypy/branch/ctypes-fix/lib_pypy/_ctypes/primitive.py pypy/branch/ctypes-fix/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py Log: Fix(?) _ensure_objects() on class _SimpleCData. Always returns self._objects, but asserts that it is None for all _SimpleCDatas except z, Z and P. Modified: pypy/branch/ctypes-fix/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/ctypes-fix/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/ctypes-fix/lib_pypy/_ctypes/primitive.py Fri Dec 10 18:03:02 2010 @@ -295,9 +295,9 @@ self.value = value def _ensure_objects(self): - if self._type_ in 'zZ': - return self._objects - return None + if self._type_ not in 'zZP': + assert self._objects is None + return self._objects def _getvalue(self): return self._buffer[0] Modified: pypy/branch/ctypes-fix/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py ============================================================================== --- pypy/branch/ctypes-fix/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py (original) +++ pypy/branch/ctypes-fix/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py Fri Dec 10 18:03:02 2010 @@ -82,8 +82,11 @@ def test_cast_functype(self): # make sure we can cast function type my_sqrt = lib.my_sqrt + saved_objects = my_sqrt._objects.copy() sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) assert sqrt(4.0) == 2.0 assert not cast(0, CFUNCTYPE(c_int)) - - + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) From afa at codespeak.net Sat Dec 11 00:33:15 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 11 Dec 2010 00:33:15 +0100 (CET) Subject: [pypy-svn] r79963 - pypy/branch/fast-forward/pypy/translator/c/src Message-ID: <20101210233315.76AA450812@codespeak.net> Author: afa Date: Sat Dec 11 00:33:13 2010 New Revision: 79963 Modified: pypy/branch/fast-forward/pypy/translator/c/src/signals.h Log: Fix tests on Windows Modified: pypy/branch/fast-forward/pypy/translator/c/src/signals.h ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/src/signals.h (original) +++ pypy/branch/fast-forward/pypy/translator/c/src/signals.h Sat Dec 11 00:33:13 2010 @@ -8,7 +8,7 @@ #include -#ifdef MS_WINDOWS +#ifdef _WIN32 #include #include #else From afa at codespeak.net Sat Dec 11 00:34:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 11 Dec 2010 00:34:43 +0100 (CET) Subject: [pypy-svn] r79964 - in pypy/branch/fast-forward/pypy: objspace/std rlib rlib/test rpython/lltypesystem rpython/module rpython/module/test translator/c/src translator/c/test Message-ID: <20101210233443.DD4A1282BAD@codespeak.net> Author: afa Date: Sat Dec 11 00:34:40 2010 New Revision: 79964 Modified: pypy/branch/fast-forward/pypy/objspace/std/complexobject.py pypy/branch/fast-forward/pypy/objspace/std/floatobject.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rlib/rmarshal.py pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py pypy/branch/fast-forward/pypy/rlib/test/test_rmarshal.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll_str.py pypy/branch/fast-forward/pypy/rpython/module/ll_strtod.py pypy/branch/fast-forward/pypy/rpython/module/test/test_ll_strtod.py pypy/branch/fast-forward/pypy/translator/c/src/ll_strtod.h pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py pypy/branch/fast-forward/pypy/translator/c/test/test_standalone.py Log: Refactor formatd() to not take a printf-like format, but several arguments instead. This prepares for the inclusion of the dtoa functions. Modified: pypy/branch/fast-forward/pypy/objspace/std/complexobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/complexobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/complexobject.py Sat Dec 11 00:34:40 2010 @@ -3,7 +3,8 @@ from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all from pypy.objspace.std.floatobject import W_FloatObject, _hash_float -from pypy.rlib.rarithmetic import formatd, isinf, isnan, copysign +from pypy.rlib.rarithmetic import ( + formatd, DTSF_STR_PRECISION, isinf, isnan, copysign) import math @@ -255,7 +256,7 @@ #w_imag = space.call_function(space.w_float,space.wrap(-w_self.imagval)) return space.newcomplex(w_self.realval,-w_self.imagval) -def format_float(x, format): +def format_float(x, code, precision): # like float2string, except that the ".0" is not necessary if isinf(x): if x > 0.0: @@ -265,12 +266,12 @@ elif isnan(x): return "nan" else: - return formatd(format, x) + return formatd(x, code, precision) def repr_format(x): - return format_float(x, "%.17g") + return format_float(x, 'r', 0) def str_format(x): - return format_float(x, "%.12g") + return format_float(x, 'g', DTSF_STR_PRECISION) def repr__Complex(space, w_complex): if w_complex.realval == 0 and copysign(1., w_complex.realval) == 1.: Modified: pypy/branch/fast-forward/pypy/objspace/std/floatobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floatobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floatobject.py Sat Dec 11 00:34:40 2010 @@ -9,7 +9,8 @@ from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.longobject import W_LongObject from pypy.rlib.rarithmetic import ovfcheck_float_to_int, intmask, isinf, isnan -from pypy.rlib.rarithmetic import formatd, LONG_BIT, INFINITY, copysign +from pypy.rlib.rarithmetic import (LONG_BIT, INFINITY, copysign, + formatd, DTSF_ADD_DOT_0, DTSF_STR_PRECISION) from pypy.rlib.rbigint import rbigint from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rfloat @@ -130,7 +131,7 @@ else: return space.wrap("0x%sp%s%d" % (s, sign, exp)) -def float2string(space, w_float, format): +def float2string(space, w_float, code, precision): x = w_float.floatval # we special-case explicitly inf and nan here if isinf(x): @@ -141,23 +142,14 @@ elif isnan(x): s = "nan" else: - s = formatd(format, x) - # We want float numbers to be recognizable as such, - # i.e., they should contain a decimal point or an exponent. - # However, %g may print the number as an integer; - # in such cases, we append ".0" to the string. - for c in s: - if c in '.eE': - break - else: - s += '.0' + s = formatd(x, code, precision, DTSF_ADD_DOT_0) return space.wrap(s) def repr__Float(space, w_float): - return float2string(space, w_float, "%.17g") + return float2string(space, w_float, 'r', 0) def str__Float(space, w_float): - return float2string(space, w_float, "%.12g") + return float2string(space, w_float, 'g', DTSF_STR_PRECISION) def format__Float_ANY(space, w_float, w_spec): return newformat.run_formatter(space, w_spec, "format_float", w_float) Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Sat Dec 11 00:34:40 2010 @@ -555,35 +555,60 @@ # float -> string -formatd_max_length = 120 +DTSF_STR_PRECISION = 12 -def formatd(fmt, x): - return fmt % (x,) +DTSF_SIGN = 0x1 +DTSF_ADD_DOT_0 = 0x2 +DTSF_ALT = 0x4 -def formatd_overflow(alt, prec, kind, x): - # msvcrt does not support the %F format. - # OTOH %F and %f only differ for 'inf' or 'nan' numbers - # which are already handled elsewhere - if kind == 'F': - kind = 'f' +DIST_FINITE = 1 +DIST_NAN = 2 +DIST_INFINITY = 3 - if ((kind in 'gG' and formatd_max_length <= 10+prec) or - (kind in 'fF' and formatd_max_length <= 53+prec)): - raise OverflowError("formatted float is too long (precision too large?)") - if alt: +formatd_ADD_DOT_0 = 0x1 + +def formatd(x, code, precision, flags=0): + "NOT_RPYTHON" + if flags & DTSF_ALT: alt = '#' else: alt = '' - fmt = "%%%s.%d%s" % (alt, prec, kind) + if code == 'r': + fmt = "%r" + else: + fmt = "%%%s.%d%s" % (alt, precision, code) + s = fmt % (x,) - return formatd(fmt, x) + if flags & formatd_ADD_DOT_0: + # We want float numbers to be recognizable as such, + # i.e., they should contain a decimal point or an exponent. + # However, %g may print the number as an integer; + # in such cases, we append ".0" to the string. + for c in s: + if c in '.eE': + break + else: + s += '.0' + elif code == 'r' and s.endswith('.0'): + s = s[:-2] -DTSF_ADD_DOT_0 = 1 + return s -DIST_FINITE = 1 -DIST_NAN = 2 -DIST_INFINITY = 3 +formatd_max_length = 120 + +def formatd_overflow(x, kind, precision, flags=0): + # msvcrt does not support the %F format. + # OTOH %F and %f only differ for 'inf' or 'nan' numbers + # which are already handled elsewhere + if kind == 'F': + kind = 'f' + + if ((kind in 'gG' and formatd_max_length < 10+precision) or + (kind in 'fF' and formatd_max_length < 53+precision)): + raise OverflowError("formatted float is too long (precision too large?)") + + return formatd(x, kind, precision, flags) def double_to_string(value, tp, precision, flags): if isnan(value): @@ -592,7 +617,7 @@ special = DIST_INFINITY else: special = DIST_FINITE - result = formatd_overflow(False, precision, tp, value) + result = formatd_overflow(value, tp, precision) return result, special # the 'float' C type Modified: pypy/branch/fast-forward/pypy/rlib/rmarshal.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rmarshal.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rmarshal.py Sat Dec 11 00:34:40 2010 @@ -195,7 +195,7 @@ def dump_float(buf, x): buf.append(TYPE_FLOAT) - s = formatd("%.17g", x) + s = formatd(x, 'g', 17) buf.append(chr(len(s))) buf += s add_dumper(annmodel.SomeFloat(), dump_float) Modified: pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/test/test_rarithmetic.py Sat Dec 11 00:34:40 2010 @@ -338,10 +338,18 @@ def test_formatd(self): from pypy.rlib.rarithmetic import formatd def f(x): - return formatd('%.2f', x) + return formatd(x, 'f', 2, 0) res = self.ll_to_string(self.interpret(f, [10/3.0])) assert res == '3.33' + def test_formatd_repr(self): + py.test.skip('WIP: Need full dtoa support to run this test') + from pypy.rlib.rarithmetic import formatd + def f(x): + return formatd(x, 'r', 0, 0) + res = self.ll_to_string(self.interpret(f, [1.1])) + assert res == '1.1' + def test_formatd_overflow(self): from pypy.translator.c.test.test_genc import compile from pypy.rlib.rarithmetic import formatd_overflow @@ -349,7 +357,7 @@ def func(x): # Test the %F format, which is not supported by # the Microsoft's msvcrt library. - return formatd_overflow(0, 4, 'F', x) + return formatd_overflow(x, 'F', 4) f = compile(func, [float]) assert f(10/3.0) == '3.3333' Modified: pypy/branch/fast-forward/pypy/rlib/test/test_rmarshal.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/test/test_rmarshal.py (original) +++ pypy/branch/fast-forward/pypy/rlib/test/test_rmarshal.py Sat Dec 11 00:34:40 2010 @@ -147,7 +147,7 @@ def f(): result = '' for num, string, fval in unmarshaller(buf): - result += '%d=%s/%s;' % (num, string, formatd('%.17g', fval)) + result += '%d=%s/%s;' % (num, string, formatd(fval, 'g', 17)) return result res = interpret(f, []) res = ''.join(res.chars) Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll_str.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll_str.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/ll_str.py Sat Dec 11 00:34:40 2010 @@ -126,5 +126,5 @@ ll_int2oct._pure_function_ = True def ll_float_str(repr, f): - return llstr(formatd("%f", f)) + return llstr(formatd(f, 'f', 6)) ll_float_str._pure_function_ = True Modified: pypy/branch/fast-forward/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/module/ll_strtod.py (original) +++ pypy/branch/fast-forward/pypy/rpython/module/ll_strtod.py Sat Dec 11 00:34:40 2010 @@ -24,17 +24,20 @@ @registering(rarithmetic.formatd) def register_formatd(self): ll_strtod = self.llexternal('LL_strtod_formatd', - [rffi.CCHARP, rffi.DOUBLE], rffi.CCHARP, + [rffi.DOUBLE, rffi.CHAR, rffi.INT], rffi.CCHARP, sandboxsafe=True, threadsafe=False) - def llimpl(fmt, x): - res = ll_strtod(fmt, x) + def llimpl(x, code, precision, flags): + if code == 'r': + code = 'g' + precision = 17 + res = ll_strtod(x, code, precision) return rffi.charp2str(res) - def oofakeimpl(fmt, x): - return ootype.oostring(rarithmetic.formatd(fmt._str, x), -1) + def oofakeimpl(x, code, precision, flags): + return ootype.oostring(rarithmetic.formatd(x, code, precision, flags), -1) - return extdef([str, float], str, 'll_strtod.ll_strtod_formatd', + return extdef([float, lltype.Char, int, int], str, 'll_strtod.ll_strtod_formatd', llimpl=llimpl, oofakeimpl=oofakeimpl, sandboxsafe=True) Modified: pypy/branch/fast-forward/pypy/rpython/module/test/test_ll_strtod.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/module/test/test_ll_strtod.py (original) +++ pypy/branch/fast-forward/pypy/rpython/module/test/test_ll_strtod.py Sat Dec 11 00:34:40 2010 @@ -6,7 +6,7 @@ class BaseTestStrtod(BaseRtypingTest): def test_formatd(self): def f(y): - return rarithmetic.formatd("%.2f", y) + return rarithmetic.formatd(y, 'f', 2) assert self.ll_to_string(self.interpret(f, [3.0])) == f(3.0) Modified: pypy/branch/fast-forward/pypy/translator/c/src/ll_strtod.h ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/src/ll_strtod.h (original) +++ pypy/branch/fast-forward/pypy/translator/c/src/ll_strtod.h Sat Dec 11 00:34:40 2010 @@ -13,7 +13,7 @@ double LL_strtod_parts_to_float(char *sign, char *beforept, char *afterpt, char *exponent); -char *LL_strtod_formatd(char *fmt, double x); +char *LL_strtod_formatd(double x, char code, int precision); /* implementations */ @@ -86,9 +86,16 @@ #define snprintf _snprintf #endif -char* LL_strtod_formatd(char *fmt, double x) { +char* LL_strtod_formatd(double x, char code, int precision) { int res; - res = snprintf(buffer, buflen, fmt, x); + const char* fmt; + if (code == 'f') fmt = "%.*f"; + else if (code == 'g') fmt = "%.*g"; + else { + strcpy(buffer, "??.?"); /* should not occur */ + return buffer; + } + res = snprintf(buffer, buflen, fmt, precision, x); if (res <= 0 || res >= buflen) { strcpy(buffer, "??.?"); /* should not occur */ } else { Modified: pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py Sat Dec 11 00:34:40 2010 @@ -283,7 +283,7 @@ def test_rarith_formatd(): from pypy.rlib.rarithmetic import formatd def fn(x): - return formatd("%.2f", x) + return formatd(x, 'f', 2, 0) f = compile(fn, [float]) Modified: pypy/branch/fast-forward/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/test/test_standalone.py Sat Dec 11 00:34:40 2010 @@ -239,7 +239,7 @@ fname2 = dirname.join("test_genc.c") fname2.write(""" void f() { - LL_strtod_formatd("%5f", 12.3); + LL_strtod_formatd(12.3, 'f', 5); }""") files = [fname, fname2] From benjamin at codespeak.net Sat Dec 11 05:46:36 2010 From: benjamin at codespeak.net (benjamin at codespeak.net) Date: Sat, 11 Dec 2010 05:46:36 +0100 (CET) Subject: [pypy-svn] r79965 - pypy/trunk/pypy/rpython/memory/gc Message-ID: <20101211044636.EC91C282B9E@codespeak.net> Author: benjamin Date: Sat Dec 11 05:46:34 2010 New Revision: 79965 Modified: pypy/trunk/pypy/rpython/memory/gc/env.py Log: provide default Modified: pypy/trunk/pypy/rpython/memory/gc/env.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gc/env.py (original) +++ pypy/trunk/pypy/rpython/memory/gc/env.py Sat Dec 11 05:46:34 2010 @@ -112,7 +112,7 @@ # ---------- Linux2 ---------- -def get_L2cache_linux2(filename): +def get_L2cache_linux2(filename="/proc/cpuinfo"): debug_start("gc-hardware") L2cache = sys.maxint try: From afa at codespeak.net Sat Dec 11 09:47:13 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 11 Dec 2010 09:47:13 +0100 (CET) Subject: [pypy-svn] r79966 - in pypy/branch/fast-forward/pypy: rlib rpython/module Message-ID: <20101211084713.4D3A5282B9E@codespeak.net> Author: afa Date: Sat Dec 11 09:47:10 2010 New Revision: 79966 Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py pypy/branch/fast-forward/pypy/rpython/module/ll_strtod.py Log: Try to fix translation Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Sat Dec 11 09:47:10 2010 @@ -567,7 +567,7 @@ formatd_ADD_DOT_0 = 0x1 -def formatd(x, code, precision, flags=0): +def _formatd(x, code, precision, flags): "NOT_RPYTHON" if flags & DTSF_ALT: alt = '#' @@ -594,6 +594,8 @@ s = s[:-2] return s +def formatd(x, code, precision, flags=0): + return _formatd(x, code, precision, flags) formatd_max_length = 120 Modified: pypy/branch/fast-forward/pypy/rpython/module/ll_strtod.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/module/ll_strtod.py (original) +++ pypy/branch/fast-forward/pypy/rpython/module/ll_strtod.py Sat Dec 11 09:47:10 2010 @@ -21,7 +21,7 @@ def __init__(self): self.configure(CConfig) - @registering(rarithmetic.formatd) + @registering(rarithmetic._formatd) def register_formatd(self): ll_strtod = self.llexternal('LL_strtod_formatd', [rffi.DOUBLE, rffi.CHAR, rffi.INT], rffi.CCHARP, From afa at codespeak.net Sat Dec 11 10:33:07 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 11 Dec 2010 10:33:07 +0100 (CET) Subject: [pypy-svn] r79967 - in pypy/branch/fast-forward/pypy: objspace/std rlib Message-ID: <20101211093307.83713282B9E@codespeak.net> Author: afa Date: Sat Dec 11 10:33:04 2010 New Revision: 79967 Modified: pypy/branch/fast-forward/pypy/objspace/std/formatting.py pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Log: More fixes Modified: pypy/branch/fast-forward/pypy/objspace/std/formatting.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/formatting.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/formatting.py Sat Dec 11 10:33:04 2010 @@ -2,7 +2,8 @@ String formatting routines. """ from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.rarithmetic import ovfcheck, formatd_overflow, isnan, isinf +from pypy.rlib.rarithmetic import ( + ovfcheck, formatd_overflow, DTSF_ALT, isnan, isinf) from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.rstring import StringBuilder, UnicodeBuilder @@ -138,8 +139,11 @@ prec = 6 if char in 'fF' and x/1e25 > 1e25: char = chr(ord(char) + 1) # 'f' => 'g' + flags = 0 + if self.f_alt: + flags |= DTSF_ALT try: - r = formatd_overflow(self.f_alt, prec, char, x) + r = formatd_overflow(x, char, prec, self.f_alt) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap( "formatted float is too long (precision too large?)")) Modified: pypy/branch/fast-forward/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rarithmetic.py Sat Dec 11 10:33:04 2010 @@ -565,8 +565,6 @@ DIST_NAN = 2 DIST_INFINITY = 3 -formatd_ADD_DOT_0 = 0x1 - def _formatd(x, code, precision, flags): "NOT_RPYTHON" if flags & DTSF_ALT: @@ -580,7 +578,7 @@ fmt = "%%%s.%d%s" % (alt, precision, code) s = fmt % (x,) - if flags & formatd_ADD_DOT_0: + if flags & DTSF_ADD_DOT_0: # We want float numbers to be recognizable as such, # i.e., they should contain a decimal point or an exponent. # However, %g may print the number as an integer; From afa at codespeak.net Sat Dec 11 11:38:51 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sat, 11 Dec 2010 11:38:51 +0100 (CET) Subject: [pypy-svn] r79968 - pypy/branch/fast-forward/pypy/objspace/std Message-ID: <20101211103851.7C554282B9E@codespeak.net> Author: afa Date: Sat Dec 11 11:38:48 2010 New Revision: 79968 Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py Log: Fix translation, maybe Modified: pypy/branch/fast-forward/pypy/objspace/std/newformat.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/newformat.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/newformat.py Sat Dec 11 11:38:48 2010 @@ -358,7 +358,7 @@ presentation_type = spec[i] if self.is_unicode: try: - the_type = spec[i].encode("ascii") + the_type = spec[i].encode("ascii")[0] except UnicodeEncodeError: raise OperationError(space.w_ValueError, space.wrap("invalid presentation type")) From arigo at codespeak.net Sat Dec 11 12:07:47 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 12:07:47 +0100 (CET) Subject: [pypy-svn] r79969 - pypy/branch/ctypes-fix/pypy/rpython/memory/gc Message-ID: <20101211110747.662FB282B9E@codespeak.net> Author: arigo Date: Sat Dec 11 12:07:45 2010 New Revision: 79969 Modified: pypy/branch/ctypes-fix/pypy/rpython/memory/gc/env.py Log: Merge fix from trunk. Modified: pypy/branch/ctypes-fix/pypy/rpython/memory/gc/env.py ============================================================================== --- pypy/branch/ctypes-fix/pypy/rpython/memory/gc/env.py (original) +++ pypy/branch/ctypes-fix/pypy/rpython/memory/gc/env.py Sat Dec 11 12:07:45 2010 @@ -112,7 +112,7 @@ # ---------- Linux2 ---------- -def get_L2cache_linux2(filename): +def get_L2cache_linux2(filename="/proc/cpuinfo"): debug_start("gc-hardware") L2cache = sys.maxint try: From arigo at codespeak.net Sat Dec 11 12:15:18 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 12:15:18 +0100 (CET) Subject: [pypy-svn] r79970 - pypy/trunk/pypy/interpreter/test Message-ID: <20101211111518.79D5B282B9E@codespeak.net> Author: arigo Date: Sat Dec 11 12:15:16 2010 New Revision: 79970 Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py Log: A failing test, causing the failure of the stdlib's test_codeop.py. Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/test/test_compiler.py Sat Dec 11 12:15:16 2010 @@ -919,3 +919,22 @@ assert e.msg == 'unindent does not match any outer indentation level' else: raise Exception("DID NOT RAISE") + + + def test_repr_vs_str(self): + source1 = "x = (\n" + source2 = "x = (\n\n" + try: + exec source1 + except SyntaxError, err1: + pass + else: + raise Exception("DID NOT RAISE") + try: + exec source2 + except SyntaxError, err2: + pass + else: + raise Exception("DID NOT RAISE") + assert str(err1) != str(err2) + assert repr(err1) != repr(err2) From arigo at codespeak.net Sat Dec 11 12:23:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 12:23:37 +0100 (CET) Subject: [pypy-svn] r79971 - in pypy/trunk/pypy: interpreter/test module/exceptions Message-ID: <20101211112337.7103B282B9E@codespeak.net> Author: arigo Date: Sat Dec 11 12:23:35 2010 New Revision: 79971 Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py pypy/trunk/pypy/module/exceptions/interp_exceptions.py Log: Fix the test with a custom __repr__ on W_SyntaxError. Modified: pypy/trunk/pypy/interpreter/test/test_compiler.py ============================================================================== --- pypy/trunk/pypy/interpreter/test/test_compiler.py (original) +++ pypy/trunk/pypy/interpreter/test/test_compiler.py Sat Dec 11 12:23:35 2010 @@ -938,3 +938,6 @@ raise Exception("DID NOT RAISE") assert str(err1) != str(err2) assert repr(err1) != repr(err2) + err3 = eval(repr(err1)) + assert str(err3) == str(err1) + assert repr(err3) == repr(err1) Modified: pypy/trunk/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/trunk/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/trunk/pypy/module/exceptions/interp_exceptions.py Sat Dec 11 12:23:35 2010 @@ -501,12 +501,29 @@ descr_str.unwrap_spec = ['self', ObjSpace] + def descr_repr(self, space): + if (len(self.args_w) == 2 + and not space.is_w(self.w_lastlineno, space.w_None) + and space.int_w(space.len(self.args_w[1])) == 4): + # fake a 5-element tuple in the repr, suitable for calling + # __init__ again + values_w = space.fixedview(self.args_w[1]) + w_tuple = space.newtuple(values_w + [self.w_lastlineno]) + args_w = [self.args_w[0], w_tuple] + args_repr = space.str_w(space.repr(space.newtuple(args_w))) + clsname = self.getclass(space).getname(space, '?') + return space.wrap(clsname + args_repr) + else: + return W_StandardError.descr_repr(self, space) + descr_repr.unwrap_spec = ['self', ObjSpace] + W_SyntaxError.typedef = TypeDef( 'SyntaxError', W_StandardError.typedef, __new__ = _new(W_SyntaxError), __init__ = interp2app(W_SyntaxError.descr_init), __str__ = interp2app(W_SyntaxError.descr_str), + __repr__ = interp2app(W_SyntaxError.descr_repr), __doc__ = W_SyntaxError.__doc__, __module__ = 'exceptions', msg = readwrite_attrproperty_w('w_msg', W_SyntaxError), From ronny at codespeak.net Sat Dec 11 12:35:13 2010 From: ronny at codespeak.net (ronny at codespeak.net) Date: Sat, 11 Dec 2010 12:35:13 +0100 (CET) Subject: [pypy-svn] r79972 - pypy/extradoc/planning/hg-migration Message-ID: <20101211113513.8FE29282B9E@codespeak.net> Author: ronny Date: Sat Dec 11 12:35:12 2010 New Revision: 79972 Modified: pypy/extradoc/planning/hg-migration/merge-howto.txt Log: update the merge howto on the goodness of hgsubversions revsets/template keywords Modified: pypy/extradoc/planning/hg-migration/merge-howto.txt ============================================================================== --- pypy/extradoc/planning/hg-migration/merge-howto.txt (original) +++ pypy/extradoc/planning/hg-migration/merge-howto.txt Sat Dec 11 12:35:12 2010 @@ -43,6 +43,23 @@ How to find the ``HR`` corresponding to ``SR`` ---------------------------------------------- + +The hgsubversion_ extension provides a revset named `svnrev`. +it provides a simple way to access svn revisions +as well as some usefull template keywords:: + + $ hg log -r 'svnrev(79389)' --template '{svnrev} {rev}:{node} {desc|firstline}\n' + 79389 38957:220cb307578d merge from trunk: svn merge svn+ssh://codespeak.net/svn/pypy/trunk -r78974:HEAD + + $ hg log -r 'svnrev(79389)' + changeset: 38957:220cb307578d + branch: jitypes2 + user: Antonio Cuni + date: Tue Nov 23 12:26:19 2010 +0000 + summary: merge from trunk: svn merge svn+ssh://codespeak.net/svn/pypy/trunk -r78974:HEAD + +Revsets can be used as argument to merge as well. + If we want to do it manually, we can use this command:: $ hg log --template '{node|short} {extras}\n' | less @@ -52,6 +69,7 @@ changeset with the highest svn revision number which is less than the target ``SR``. + Alternatively, we can use the svnup_ mercurial extension. We need to put these lines into ``~/.hgrc``:: @@ -66,5 +84,6 @@ date: Tue Nov 23 07:02:43 2010 +0000 summary: Comment out non-jit builds to not overload the server +.. _hgsubversion: http://bitbucket.org/durin42/hgsubversion .. http://codespeak.net/svn/user/antocuni/hg-conversion/svnup.py From arigo at codespeak.net Sat Dec 11 14:39:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 14:39:51 +0100 (CET) Subject: [pypy-svn] r79973 - pypy/trunk/pypy/rpython/memory/gctransform Message-ID: <20101211133951.01710282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 14:39:49 2010 New Revision: 79973 Modified: pypy/trunk/pypy/rpython/memory/gctransform/asmgcroot.py Log: Tentative fix for a rare case that occurred so far only on Windows. Modified: pypy/trunk/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/trunk/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/trunk/pypy/rpython/memory/gctransform/asmgcroot.py Sat Dec 11 14:39:49 2010 @@ -306,6 +306,16 @@ if item: self._shape_decompressor.setaddr(item) return + # there is a rare risk that the array contains *two* entries + # with the same key, one of which is dead (null value), and we + # found the dead one above. Solve this case by replacing all + # dead keys with nulls, sorting again, and then trying again. + replace_dead_entries_with_nulls(gcmapstart2, gcmapend2) + sort_gcmap(gcmapstart2, gcmapend2) + item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) + if item: + self._shape_decompressor.setaddr(item) + return # the item may have been not found because the main array was # not sorted. Sort it and try again. win32_follow_gcmap_jmp(gcmapstart, gcmapend) @@ -401,6 +411,15 @@ rffi.cast(rffi.SIZE_T, arrayitemsize), llhelper(QSORT_CALLBACK_PTR, _compare_gcmap_entries)) +def replace_dead_entries_with_nulls(start, end): + # replace the dead entries (null value) with a null key. + count = (end - start) // arrayitemsize - 1 + while count >= 0: + item = start + count * arrayitemsize + if item.address[1] == llmemory.NULL: + item.address[0] = llmemory.NULL + count -= 1 + if sys.platform == 'win32': def win32_follow_gcmap_jmp(start, end): # The initial gcmap table contains addresses to a JMP From arigo at codespeak.net Sat Dec 11 14:47:19 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 14:47:19 +0100 (CET) Subject: [pypy-svn] r79974 - in pypy/trunk: lib_pypy/_ctypes pypy/module/test_lib_pypy/ctypes_tests Message-ID: <20101211134719.0D67C282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 14:47:17 2010 New Revision: 79974 Modified: pypy/trunk/lib_pypy/_ctypes/primitive.py pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py Log: Merge from branch/ctypes-fix: Fix _ensure_objects() on class _SimpleCData. Always returns self._objects, but asserts that it is None for all _SimpleCDatas except z, Z and P. Modified: pypy/trunk/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/trunk/lib_pypy/_ctypes/primitive.py (original) +++ pypy/trunk/lib_pypy/_ctypes/primitive.py Sat Dec 11 14:47:17 2010 @@ -295,9 +295,9 @@ self.value = value def _ensure_objects(self): - if self._type_ in 'zZ': - return self._objects - return None + if self._type_ not in 'zZP': + assert self._objects is None + return self._objects def _getvalue(self): return self._buffer[0] Modified: pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py ============================================================================== --- pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py (original) +++ pypy/trunk/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py Sat Dec 11 14:47:17 2010 @@ -82,8 +82,11 @@ def test_cast_functype(self): # make sure we can cast function type my_sqrt = lib.my_sqrt + saved_objects = my_sqrt._objects.copy() sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) assert sqrt(4.0) == 2.0 assert not cast(0, CFUNCTYPE(c_int)) - - + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) From arigo at codespeak.net Sat Dec 11 14:47:34 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 14:47:34 +0100 (CET) Subject: [pypy-svn] r79975 - pypy/branch/ctypes-fix Message-ID: <20101211134734.48FD0282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 14:47:32 2010 New Revision: 79975 Removed: pypy/branch/ctypes-fix/ Log: Remove merged branch. From arigo at codespeak.net Sat Dec 11 15:06:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 15:06:46 +0100 (CET) Subject: [pypy-svn] r79976 - pypy/extradoc/planning Message-ID: <20101211140646.7003E282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 15:06:44 2010 New Revision: 79976 Modified: pypy/extradoc/planning/1.4.1.txt Log: Update. Modified: pypy/extradoc/planning/1.4.1.txt ============================================================================== --- pypy/extradoc/planning/1.4.1.txt (original) +++ pypy/extradoc/planning/1.4.1.txt Sat Dec 11 15:06:44 2010 @@ -9,16 +9,18 @@ * Improve the performance of the ``binascii`` module, and of ``hashlib.md5`` and ``hashlib.sha``. -* Fix a corner case in the GC (minimark). +* Fix two corner cases in the GC (one in minimark, one in asmgcc+JIT). * Fix cpyext on Mac OS X. (Loading C extension modules in PyPy is still considered *alpha stage!*) -* Fixed a corner case in the JIT, leading to "Fatal RPython error: - AssertionError". +* Fixed a corner case in the JIT's optimizer, leading to "Fatal RPython + error: AssertionError". * Added some missing functions from the 'os' module. +* Fix ctypes (it was not propagating keepalive information from c_void_p). + Plan: @@ -27,9 +29,3 @@ * Merge jit-unroll-loops (?) * Migrate to mercurial - -* Finish looking in the ctypes bug:: - - c = cast(cast(c_char_p("hello"), c_void_p), c_char_p) - gc.collect() - c.value # garbage From hakanardo at codespeak.net Sat Dec 11 15:10:31 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 11 Dec 2010 15:10:31 +0100 (CET) Subject: [pypy-svn] r79977 - in pypy/branch/jit-unroll-loops: . ctypes_configure lib-python lib-python/modified-2.5.2/distutils lib-python/modified-2.5.2/distutils/tests lib-python/modified-2.5.2/encodings lib-python/modified-2.5.2/test lib-python/modified-2.5.2/test/output lib_pypy lib_pypy/_ctypes lib_pypy/pypy_test pypy pypy/annotation pypy/annotation/test pypy/config pypy/config/test pypy/doc pypy/doc/config pypy/doc/statistic pypy/interpreter pypy/interpreter/pyparser pypy/interpreter/pyparser/test pypy/interpreter/test pypy/jit pypy/jit/backend pypy/jit/backend/llgraph pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/codewriter pypy/jit/codewriter/test pypy/jit/metainterp pypy/jit/metainterp/optimizeopt pypy/jit/metainterp/test pypy/jit/tl pypy/jit/tl/spli pypy/jit/tool pypy/jit/tool/test pypy/module/__pypy__ pypy/module/__pypy__/test pypy/module/_lsprof pypy/module/_lsprof/test pypy/module/_minimal_curses pypy/module/_pickle_support pypy/module/_stackless pypy/module/array/benchmark pypy/module/array/test pypy/module/binascii pypy/module/binascii/test pypy/module/cpyext pypy/module/cpyext/include pypy/module/cpyext/src pypy/module/cpyext/test pypy/module/exceptions pypy/module/fcntl/test pypy/module/imp pypy/module/itertools pypy/module/posix pypy/module/posix/test pypy/module/pyexpat pypy/module/pyexpat/test pypy/module/pypyjit pypy/module/pypyjit/test pypy/module/select pypy/module/sys pypy/module/sys/test pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rlib/rsre pypy/rlib/rsre/test pypy/rlib/test pypy/rpython pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/memory/gc pypy/rpython/memory/gc/test pypy/rpython/memory/gctransform pypy/rpython/memory/test pypy/rpython/module pypy/rpython/ootypesystem pypy/rpython/test pypy/tool pypy/tool/release pypy/tool/release/test pypy/translator pypy/translator/c pypy/translator/c/gcc pypy/translator/c/gcc/test pypy/translator/c/src pypy/translator/c/test pypy/translator/goal pypy/translator/goal/test2 pypy/translator/platform pypy/translator/sandbox/test pypy/translator/tool site-packages Message-ID: <20101211141031.C1F26282B9C@codespeak.net> Author: hakanardo Date: Sat Dec 11 15:10:15 2010 New Revision: 79977 Added: pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/seq_tests.py - copied unchanged from r79975, pypy/trunk/lib-python/modified-2.5.2/test/seq_tests.py pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/test_eof.py - copied unchanged from r79975, pypy/trunk/lib-python/modified-2.5.2/test/test_eof.py pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/hack___pypy__.py - copied unchanged from r79975, pypy/trunk/lib_pypy/pypy_test/hack___pypy__.py pypy/branch/jit-unroll-loops/pypy/config/support.py - copied unchanged from r79975, pypy/trunk/pypy/config/support.py pypy/branch/jit-unroll-loops/pypy/config/test/test_support.py - copied unchanged from r79975, pypy/trunk/pypy/config/test/test_support.py pypy/branch/jit-unroll-loops/pypy/doc/config/objspace.usemodules.binascii.txt - copied unchanged from r79975, pypy/trunk/pypy/doc/config/objspace.usemodules.binascii.txt pypy/branch/jit-unroll-loops/pypy/doc/config/translation.jit_ffi.txt - copied unchanged from r79975, pypy/trunk/pypy/doc/config/translation.jit_ffi.txt pypy/branch/jit-unroll-loops/pypy/doc/release-1.4.0beta.txt - copied unchanged from r79975, pypy/trunk/pypy/doc/release-1.4.0beta.txt pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/asmmemmgr.py - copied unchanged from r79975, pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/test/test_asmmemmgr.py - copied unchanged from r79975, pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/codebuf.py - copied unchanged from r79975, pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/memmgr.py - copied unchanged from r79975, pypy/trunk/pypy/jit/metainterp/memmgr.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_memmgr.py - copied unchanged from r79975, pypy/trunk/pypy/jit/metainterp/test/test_memmgr.py pypy/branch/jit-unroll-loops/pypy/jit/tool/cpython.vmrss - copied unchanged from r79975, pypy/trunk/pypy/jit/tool/cpython.vmrss pypy/branch/jit-unroll-loops/pypy/jit/tool/log-template.gnumeric - copied unchanged from r79975, pypy/trunk/pypy/jit/tool/log-template.gnumeric pypy/branch/jit-unroll-loops/pypy/jit/tool/log2gnumeric.py - copied unchanged from r79975, pypy/trunk/pypy/jit/tool/log2gnumeric.py pypy/branch/jit-unroll-loops/pypy/jit/tool/test/test_log2gnumeric.py - copied unchanged from r79975, pypy/trunk/pypy/jit/tool/test/test_log2gnumeric.py pypy/branch/jit-unroll-loops/pypy/jit/tool/test/test_loopcounter.py - copied unchanged from r79975, pypy/trunk/pypy/jit/tool/test/test_loopcounter.py pypy/branch/jit-unroll-loops/pypy/module/__pypy__/interp_debug.py - copied unchanged from r79975, pypy/trunk/pypy/module/__pypy__/interp_debug.py pypy/branch/jit-unroll-loops/pypy/module/__pypy__/test/test_debug.py - copied unchanged from r79975, pypy/trunk/pypy/module/__pypy__/test/test_debug.py pypy/branch/jit-unroll-loops/pypy/module/binascii/ (props changed) - copied from r79975, pypy/trunk/pypy/module/binascii/ pypy/branch/jit-unroll-loops/pypy/module/cpyext/include/fileobject.h - copied unchanged from r79975, pypy/trunk/pypy/module/cpyext/include/fileobject.h pypy/branch/jit-unroll-loops/pypy/module/cpyext/include/structseq.h - copied unchanged from r79975, pypy/trunk/pypy/module/cpyext/include/structseq.h pypy/branch/jit-unroll-loops/pypy/module/cpyext/pypyintf.py - copied unchanged from r79975, pypy/trunk/pypy/module/cpyext/pypyintf.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/src/structseq.c - copied unchanged from r79975, pypy/trunk/pypy/module/cpyext/src/structseq.c pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_structseq.py - copied unchanged from r79975, pypy/trunk/pypy/module/cpyext/test/test_structseq.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/env.py - copied unchanged from r79975, pypy/trunk/pypy/rpython/memory/gc/env.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/test/test_env.py - copied unchanged from r79975, pypy/trunk/pypy/rpython/memory/gc/test/test_env.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/test/test_inspector.py - copied unchanged from r79975, pypy/trunk/pypy/rpython/memory/gc/test/test_inspector.py pypy/branch/jit-unroll-loops/pypy/tool/debug_print.py - copied unchanged from r79975, pypy/trunk/pypy/tool/debug_print.py pypy/branch/jit-unroll-loops/pypy/translator/c/src/asm_gcc_x86_64.h - copied unchanged from r79975, pypy/trunk/pypy/translator/c/src/asm_gcc_x86_64.h pypy/branch/jit-unroll-loops/pypy/translator/platform/freebsd.py - copied unchanged from r79975, pypy/trunk/pypy/translator/platform/freebsd.py Removed: pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/encodings/ pypy/branch/jit-unroll-loops/pypy/doc/config/translation.jit_debug.txt pypy/branch/jit-unroll-loops/pypy/translator/platform/freebsd7.py Modified: pypy/branch/jit-unroll-loops/ (props changed) pypy/branch/jit-unroll-loops/ctypes_configure/configure.py pypy/branch/jit-unroll-loops/lib-python/conftest.py pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/msvccompiler.py pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/unixccompiler.py pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/output/test_cProfile pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/test_genexps.py pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/array.py pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/basics.py pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/builtin.py pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/function.py pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/pointer.py pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/primitive.py pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/structure.py pypy/branch/jit-unroll-loops/lib_pypy/_hashlib.py pypy/branch/jit-unroll-loops/lib_pypy/_locale.py pypy/branch/jit-unroll-loops/lib_pypy/_marshal.py pypy/branch/jit-unroll-loops/lib_pypy/_minimal_curses.py pypy/branch/jit-unroll-loops/lib_pypy/_pypy_interact.py pypy/branch/jit-unroll-loops/lib_pypy/binascii.py pypy/branch/jit-unroll-loops/lib_pypy/cPickle.py pypy/branch/jit-unroll-loops/lib_pypy/cmath.py pypy/branch/jit-unroll-loops/lib_pypy/ctypes_support.py pypy/branch/jit-unroll-loops/lib_pypy/grp.py pypy/branch/jit-unroll-loops/lib_pypy/hashlib.py pypy/branch/jit-unroll-loops/lib_pypy/itertools.py pypy/branch/jit-unroll-loops/lib_pypy/msvcrt.py pypy/branch/jit-unroll-loops/lib_pypy/pwd.py pypy/branch/jit-unroll-loops/lib_pypy/pyexpat.py pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/test_hashlib.py pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/test_structseq.py pypy/branch/jit-unroll-loops/lib_pypy/readline.py pypy/branch/jit-unroll-loops/lib_pypy/resource.py pypy/branch/jit-unroll-loops/lib_pypy/syslog.py pypy/branch/jit-unroll-loops/pypy/ (props changed) pypy/branch/jit-unroll-loops/pypy/annotation/annrpython.py pypy/branch/jit-unroll-loops/pypy/annotation/binaryop.py pypy/branch/jit-unroll-loops/pypy/annotation/bookkeeper.py pypy/branch/jit-unroll-loops/pypy/annotation/description.py pypy/branch/jit-unroll-loops/pypy/annotation/listdef.py pypy/branch/jit-unroll-loops/pypy/annotation/model.py pypy/branch/jit-unroll-loops/pypy/annotation/specialize.py pypy/branch/jit-unroll-loops/pypy/annotation/test/test_annrpython.py pypy/branch/jit-unroll-loops/pypy/annotation/unaryop.py pypy/branch/jit-unroll-loops/pypy/config/pypyoption.py pypy/branch/jit-unroll-loops/pypy/config/translationoption.py pypy/branch/jit-unroll-loops/pypy/conftest.py pypy/branch/jit-unroll-loops/pypy/doc/cpython_differences.txt pypy/branch/jit-unroll-loops/pypy/doc/faq.txt pypy/branch/jit-unroll-loops/pypy/doc/index.txt pypy/branch/jit-unroll-loops/pypy/doc/release-1.4.0.txt pypy/branch/jit-unroll-loops/pypy/doc/sprint-reports.txt pypy/branch/jit-unroll-loops/pypy/doc/statistic/release_dates.dat pypy/branch/jit-unroll-loops/pypy/doc/statistic/sprint_dates.dat pypy/branch/jit-unroll-loops/pypy/interpreter/argument.py pypy/branch/jit-unroll-loops/pypy/interpreter/baseobjspace.py pypy/branch/jit-unroll-loops/pypy/interpreter/executioncontext.py pypy/branch/jit-unroll-loops/pypy/interpreter/function.py pypy/branch/jit-unroll-loops/pypy/interpreter/gateway.py pypy/branch/jit-unroll-loops/pypy/interpreter/generator.py pypy/branch/jit-unroll-loops/pypy/interpreter/mixedmodule.py pypy/branch/jit-unroll-loops/pypy/interpreter/pycode.py pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/error.py pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/pytokenizer.py pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/test/test_pyparse.py pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_argument.py pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_compiler.py pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_executioncontext.py pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_function.py pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_gateway.py pypy/branch/jit-unroll-loops/pypy/jit/backend/detect_cpu.py pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/llimpl.py pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/runner.py pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/gc.py pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/llmodel.py pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/test/test_gc.py pypy/branch/jit-unroll-loops/pypy/jit/backend/model.py pypy/branch/jit-unroll-loops/pypy/jit/backend/test/runner_test.py pypy/branch/jit-unroll-loops/pypy/jit/backend/test/test_random.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/assembler.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/regalloc.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/regloc.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/runner.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/rx86.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/support.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_assembler.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_gc_integration.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regalloc.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regalloc2.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regloc.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_runner.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_rx86.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_zll_random.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_zmath.py pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/valgrind.py pypy/branch/jit-unroll-loops/pypy/jit/codewriter/call.py pypy/branch/jit-unroll-loops/pypy/jit/codewriter/jtransform.py pypy/branch/jit-unroll-loops/pypy/jit/codewriter/regalloc.py pypy/branch/jit-unroll-loops/pypy/jit/codewriter/support.py pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_codewriter.py pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_list.py pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_regalloc.py pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_void_list.py pypy/branch/jit-unroll-loops/pypy/jit/conftest.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/history.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/jitprof.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/__init__.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/optimizer.py (contents, props changed) pypy/branch/jit-unroll-loops/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_compile.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_exception.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_fficall.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_list.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_logger.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizefficall.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_pyjitpl.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_recursive.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_send_nounroll.py (props changed) pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_virtualizable.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_warmspot.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_warmstate.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmspot.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmstate.py pypy/branch/jit-unroll-loops/pypy/jit/tl/spli/interpreter.py pypy/branch/jit-unroll-loops/pypy/jit/tl/tl.py pypy/branch/jit-unroll-loops/pypy/jit/tool/jitoutput.py pypy/branch/jit-unroll-loops/pypy/jit/tool/loopcounter.py pypy/branch/jit-unroll-loops/pypy/jit/tool/test/test_jitoutput.py pypy/branch/jit-unroll-loops/pypy/module/__pypy__/__init__.py pypy/branch/jit-unroll-loops/pypy/module/__pypy__/interp_magic.py pypy/branch/jit-unroll-loops/pypy/module/__pypy__/test/test_special.py pypy/branch/jit-unroll-loops/pypy/module/_lsprof/interp_lsprof.py pypy/branch/jit-unroll-loops/pypy/module/_lsprof/test/test_cprofile.py pypy/branch/jit-unroll-loops/pypy/module/_minimal_curses/__init__.py pypy/branch/jit-unroll-loops/pypy/module/_pickle_support/maker.py pypy/branch/jit-unroll-loops/pypy/module/_stackless/interp_coroutine.py pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/test/test_array.py pypy/branch/jit-unroll-loops/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/jit-unroll-loops/pypy/module/binascii/test/ (props changed) pypy/branch/jit-unroll-loops/pypy/module/cpyext/__init__.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/api.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/cdatetime.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/include/Python.h pypy/branch/jit-unroll-loops/pypy/module/cpyext/intobject.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/object.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/pyerrors.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/pythonrun.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/sequence.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/slotdefs.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/src/getargs.c pypy/branch/jit-unroll-loops/pypy/module/cpyext/state.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/stubs.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_arraymodule.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_cpyext.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_datetime.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_intobject.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_object.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_pyerrors.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_sequence.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_typeobject.py pypy/branch/jit-unroll-loops/pypy/module/cpyext/typeobject.py pypy/branch/jit-unroll-loops/pypy/module/exceptions/interp_exceptions.py pypy/branch/jit-unroll-loops/pypy/module/fcntl/test/test_fcntl.py pypy/branch/jit-unroll-loops/pypy/module/imp/importing.py pypy/branch/jit-unroll-loops/pypy/module/itertools/interp_itertools.py pypy/branch/jit-unroll-loops/pypy/module/posix/__init__.py pypy/branch/jit-unroll-loops/pypy/module/posix/interp_posix.py pypy/branch/jit-unroll-loops/pypy/module/posix/test/test_posix2.py pypy/branch/jit-unroll-loops/pypy/module/pyexpat/interp_pyexpat.py pypy/branch/jit-unroll-loops/pypy/module/pyexpat/test/test_parser.py pypy/branch/jit-unroll-loops/pypy/module/pypyjit/__init__.py pypy/branch/jit-unroll-loops/pypy/module/pypyjit/policy.py pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_policy.py pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_pypy_c.py pypy/branch/jit-unroll-loops/pypy/module/select/interp_select.py pypy/branch/jit-unroll-loops/pypy/module/sys/__init__.py pypy/branch/jit-unroll-loops/pypy/module/sys/state.py pypy/branch/jit-unroll-loops/pypy/module/sys/test/test_initialpath.py pypy/branch/jit-unroll-loops/pypy/module/sys/vm.py pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py pypy/branch/jit-unroll-loops/pypy/objspace/std/complexobject.py pypy/branch/jit-unroll-loops/pypy/objspace/std/complextype.py pypy/branch/jit-unroll-loops/pypy/objspace/std/floattype.py pypy/branch/jit-unroll-loops/pypy/objspace/std/longobject.py pypy/branch/jit-unroll-loops/pypy/objspace/std/mapdict.py pypy/branch/jit-unroll-loops/pypy/objspace/std/objspace.py pypy/branch/jit-unroll-loops/pypy/objspace/std/strutil.py pypy/branch/jit-unroll-loops/pypy/objspace/std/test/test_complexobject.py pypy/branch/jit-unroll-loops/pypy/objspace/std/test/test_strutil.py pypy/branch/jit-unroll-loops/pypy/rlib/debug.py pypy/branch/jit-unroll-loops/pypy/rlib/jit.py pypy/branch/jit-unroll-loops/pypy/rlib/libffi.py pypy/branch/jit-unroll-loops/pypy/rlib/rarithmetic.py pypy/branch/jit-unroll-loops/pypy/rlib/rdynload.py pypy/branch/jit-unroll-loops/pypy/rlib/rerased.py (contents, props changed) pypy/branch/jit-unroll-loops/pypy/rlib/rmmap.py pypy/branch/jit-unroll-loops/pypy/rlib/rsre/rsre_core.py pypy/branch/jit-unroll-loops/pypy/rlib/rsre/test/test_zjit.py pypy/branch/jit-unroll-loops/pypy/rlib/test/test_debug.py pypy/branch/jit-unroll-loops/pypy/rlib/test/test_libffi.py pypy/branch/jit-unroll-loops/pypy/rlib/test/test_rerased.py (props changed) pypy/branch/jit-unroll-loops/pypy/rpython/extfunc.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/ll2ctypes.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/llarena.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/llmemory.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/lltype.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rbuiltin.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rclass.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rdict.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rpbc.py pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/test/test_llarena.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/base.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/generation.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/inspector.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/markcompact.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/minimark.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/semispace.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/test/test_direct.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/boehm.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/framework.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/refcounting.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/support.py pypy/branch/jit-unroll-loops/pypy/rpython/memory/test/test_gc.py pypy/branch/jit-unroll-loops/pypy/rpython/module/ll_os.py pypy/branch/jit-unroll-loops/pypy/rpython/module/ll_time.py pypy/branch/jit-unroll-loops/pypy/rpython/ootypesystem/rpbc.py pypy/branch/jit-unroll-loops/pypy/rpython/rlist.py pypy/branch/jit-unroll-loops/pypy/rpython/rpbc.py pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rclass.py pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rint.py pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rlist.py pypy/branch/jit-unroll-loops/pypy/tool/ansi_print.py pypy/branch/jit-unroll-loops/pypy/tool/error.py pypy/branch/jit-unroll-loops/pypy/tool/logparser.py pypy/branch/jit-unroll-loops/pypy/tool/release/force-builds.py pypy/branch/jit-unroll-loops/pypy/tool/release/make_release.py pypy/branch/jit-unroll-loops/pypy/tool/release/package.py pypy/branch/jit-unroll-loops/pypy/tool/release/test/test_package.py pypy/branch/jit-unroll-loops/pypy/tool/terminal.py pypy/branch/jit-unroll-loops/pypy/translator/c/funcgen.py pypy/branch/jit-unroll-loops/pypy/translator/c/gcc/test/test_trackgcroot.py pypy/branch/jit-unroll-loops/pypy/translator/c/gcc/trackgcroot.py pypy/branch/jit-unroll-loops/pypy/translator/c/src/debug_alloc.h pypy/branch/jit-unroll-loops/pypy/translator/c/src/debug_print.h pypy/branch/jit-unroll-loops/pypy/translator/c/src/g_include.h pypy/branch/jit-unroll-loops/pypy/translator/c/src/mem.h pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_extfunc.py pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_newgc.py pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_standalone.py pypy/branch/jit-unroll-loops/pypy/translator/driver.py pypy/branch/jit-unroll-loops/pypy/translator/goal/app_main.py pypy/branch/jit-unroll-loops/pypy/translator/goal/targetpypystandalone.py pypy/branch/jit-unroll-loops/pypy/translator/goal/test2/test_app_main.py pypy/branch/jit-unroll-loops/pypy/translator/platform/__init__.py pypy/branch/jit-unroll-loops/pypy/translator/platform/darwin.py pypy/branch/jit-unroll-loops/pypy/translator/platform/linux.py pypy/branch/jit-unroll-loops/pypy/translator/platform/posix.py pypy/branch/jit-unroll-loops/pypy/translator/sandbox/test/test_sandbox.py pypy/branch/jit-unroll-loops/pypy/translator/tool/cbuild.py pypy/branch/jit-unroll-loops/pypy/translator/tool/reftracker.py pypy/branch/jit-unroll-loops/site-packages/ (props changed) Log: svn merge -r79225:HEAD svn+ssh://hakanardo at codespeak.net/svn/pypy/trunk Modified: pypy/branch/jit-unroll-loops/ctypes_configure/configure.py ============================================================================== --- pypy/branch/jit-unroll-loops/ctypes_configure/configure.py (original) +++ pypy/branch/jit-unroll-loops/ctypes_configure/configure.py Sat Dec 11 15:10:15 2010 @@ -559,6 +559,7 @@ C_HEADER = """ #include #include /* for offsetof() */ +#include /* FreeBSD: for uint64_t */ void dump(char* key, int value) { printf("%s: %d\\n", key, value); Modified: pypy/branch/jit-unroll-loops/lib-python/conftest.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib-python/conftest.py (original) +++ pypy/branch/jit-unroll-loops/lib-python/conftest.py Sat Dec 11 15:10:15 2010 @@ -139,7 +139,7 @@ RegrTest('test_augassign.py', core=True), RegrTest('test_base64.py'), RegrTest('test_bastion.py'), - RegrTest('test_binascii.py'), + RegrTest('test_binascii.py', usemodules='binascii'), RegrTest('test_binhex.py'), Modified: pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/msvccompiler.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/msvccompiler.py (original) +++ pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/msvccompiler.py Sat Dec 11 15:10:15 2010 @@ -25,17 +25,20 @@ from distutils import log from distutils.util import get_platform -import _winreg - -RegOpenKeyEx = _winreg.OpenKeyEx -RegEnumKey = _winreg.EnumKey -RegEnumValue = _winreg.EnumValue -RegError = _winreg.error - -HKEYS = (_winreg.HKEY_USERS, - _winreg.HKEY_CURRENT_USER, - _winreg.HKEY_LOCAL_MACHINE, - _winreg.HKEY_CLASSES_ROOT) +try: + import _winreg +except ImportError: + pass +else: + RegOpenKeyEx = _winreg.OpenKeyEx + RegEnumKey = _winreg.EnumKey + RegEnumValue = _winreg.EnumValue + RegError = _winreg.error + + HKEYS = (_winreg.HKEY_USERS, + _winreg.HKEY_CURRENT_USER, + _winreg.HKEY_LOCAL_MACHINE, + _winreg.HKEY_CLASSES_ROOT) VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" VSEXPRESS_BASE = r"Software\Microsoft\VCExpress\%0.1f" Modified: pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py (original) +++ pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/tests/test_msvccompiler.py Sat Dec 11 15:10:15 2010 @@ -11,11 +11,15 @@ else: self.fail("could not find a suitable manifest") +class MsvcCompilerSimplerTestCase(unittest.TestCase): + def test_import_module(self): + from distutils.msvccompiler import MSVCCompiler + def test_suite(): if sys.platform == 'win32': return unittest.makeSuite(MsvcCompilerTestCase) else: - return unittest.TestSuite([]) + return unittest.makeSuite(MsvcCompilerSimplerTestCase) if __name__ == "__main__": unittest.main(defaultTest="test_suite") Modified: pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/unixccompiler.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/unixccompiler.py (original) +++ pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/distutils/unixccompiler.py Sat Dec 11 15:10:15 2010 @@ -121,7 +121,22 @@ } if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() executables['ranlib'] = ["ranlib"] + executables['linker_so'] += ['-undefined', 'dynamic_lookup'] + + for k, v in executables.iteritems(): + if v and v[0] == 'cc': + v += ['-arch', arch] + # Needed for the filename generation methods provided by the base # class, CCompiler. NB. whoever instantiates/uses a particular Modified: pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/output/test_cProfile ============================================================================== --- pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/output/test_cProfile (original) +++ pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/output/test_cProfile Sat Dec 11 15:10:15 2010 @@ -14,66 +14,66 @@ 4 0.116 0.029 0.120 0.030 test_cProfile.py:78(helper1) 2 0.000 0.000 0.140 0.070 test_cProfile.py:89(helper2_indirect) 8 0.312 0.039 0.400 0.050 test_cProfile.py:93(helper2) - 4 0.000 0.000 0.000 0.000 {append} - 1 0.000 0.000 0.000 0.000 {disable} 12 0.000 0.000 0.012 0.001 {hasattr} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} + 1 0.000 0.000 0.000 0.000 {method 'disable' of 'Profile' objects} 8 0.000 0.000 0.000 0.000 {range} 4 0.000 0.000 0.000 0.000 {sys.exc_info} Ordered by: standard name -Function called... - ncalls tottime cumtime -:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) -test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) - 8 0.000 0.000 {range} -test_cProfile.py:115(__getattr__) -> -test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) - 2 0.040 0.600 test_cProfile.py:60(helper) -test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 20 0.020 0.020 test_cProfile.py:53(mul) -test_cProfile.py:53(mul) -> -test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) - 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) - 6 0.234 0.300 test_cProfile.py:93(helper2) -test_cProfile.py:78(helper1) -> 4 0.000 0.000 {append} - 4 0.000 0.004 {hasattr} - 4 0.000 0.000 {sys.exc_info} -test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) - 2 0.078 0.100 test_cProfile.py:93(helper2) -test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) - 8 0.000 0.008 {hasattr} -{append} -> -{disable} -> -{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) -{range} -> -{sys.exc_info} -> +Function called... + ncalls tottime cumtime +:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) +test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) + 8 0.000 0.000 {range} +test_cProfile.py:115(__getattr__) -> +test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) + 2 0.040 0.600 test_cProfile.py:60(helper) +test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 20 0.020 0.020 test_cProfile.py:53(mul) +test_cProfile.py:53(mul) -> +test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) + 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) + 6 0.234 0.300 test_cProfile.py:93(helper2) +test_cProfile.py:78(helper1) -> 4 0.000 0.004 {hasattr} + 4 0.000 0.000 {method 'append' of 'list' objects} + 4 0.000 0.000 {sys.exc_info} +test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) + 2 0.078 0.100 test_cProfile.py:93(helper2) +test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) + 8 0.000 0.008 {hasattr} +{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) +{method 'append' of 'list' objects} -> +{method 'disable' of 'Profile' objects} -> +{range} -> +{sys.exc_info} -> Ordered by: standard name -Function was called by... - ncalls tottime cumtime -:1() <- -test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) -test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) - 12 0.012 0.012 {hasattr} -test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() -test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) - 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) -test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) -test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) -test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) -test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) -test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) - 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) -{append} <- 4 0.000 0.000 test_cProfile.py:78(helper1) -{disable} <- -{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) - 8 0.000 0.008 test_cProfile.py:93(helper2) -{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) -{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +Function was called by... + ncalls tottime cumtime +:1() <- +test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) +test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) + 12 0.012 0.012 {hasattr} +test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() +test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) + 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) +test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) +test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) +test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) +test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) +test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) + 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) +{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) + 8 0.000 0.008 test_cProfile.py:93(helper2) +{method 'append' of 'list' objects} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +{method 'disable' of 'Profile' objects} <- +{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) +{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) Modified: pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/test_genexps.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/test_genexps.py (original) +++ pypy/branch/jit-unroll-loops/lib-python/modified-2.5.2/test/test_genexps.py Sat Dec 11 15:10:15 2010 @@ -77,10 +77,10 @@ Verify that parenthesis are required in a statement >>> def f(n): - ... return i*i for i in xrange(n) + ... return i*i for i in xrange(n) #doctest: +ELLIPSIS Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax... Verify that parenthesis are required when used as a keyword argument value Modified: pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/array.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/array.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/array.py Sat Dec 11 15:10:15 2010 @@ -99,6 +99,9 @@ if len(value) > self._length_: raise ValueError("Invalid length") value = self(*value) + elif not isinstance(value, self): + raise TypeError("expected string or Unicode object, %s found" + % (value.__class__.__name__,)) else: if isinstance(value, tuple): if len(value) > self._length_: @@ -107,22 +110,43 @@ return _CDataMeta.from_param(self, value) def array_get_slice_params(self, index): - if index.step is not None: - raise TypeError("3 arg slices not supported (for no reason)") - start = index.start or 0 - stop = index.stop or self._length_ - return start, stop + if hasattr(self, '_length_'): + start, stop, step = index.indices(self._length_) + else: + step = index.step + if step is None: + step = 1 + start = index.start + stop = index.stop + if start is None: + if step > 0: + start = 0 + else: + raise ValueError("slice start is required for step < 0") + if stop is None: + raise ValueError("slice stop is required") + + return start, stop, step def array_slice_setitem(self, index, value): - start, stop = self._get_slice_params(index) - if stop - start != len(value): + start, stop, step = self._get_slice_params(index) + + if ((step < 0 and stop >= start) or + (step > 0 and start >= stop)): + slicelength = 0 + elif step < 0: + slicelength = (stop - start + 1) / step + 1 + else: + slicelength = (stop - start - 1) / step + 1; + + if slicelength != len(value): raise ValueError("Can only assign slices of the same length") - for i in range(start, stop): - self[i] = value[i - start] + for i, j in enumerate(range(start, stop, step)): + self[j] = value[i] def array_slice_getitem(self, index): - start, stop = self._get_slice_params(index) - l = [self[i] for i in range(start, stop)] + start, stop, step = self._get_slice_params(index) + l = [self[i] for i in range(start, stop, step)] letter = getattr(self._type_, '_type_', None) if letter == 'c': return "".join(l) @@ -135,7 +159,8 @@ _ffiargshape = 'P' def __init__(self, *args): - self._buffer = self._ffiarray(self._length_, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(self._length_, autofree=True) for i, arg in enumerate(args): self[i] = arg @@ -162,9 +187,10 @@ self._slice_setitem(index, value) return index = self._fix_index(index) - if ensure_objects(value) is not None: - store_reference(self, index, value._objects) - arg = self._type_._CData_value(value) + cobj = self._type_.from_param(value) + if ensure_objects(cobj) is not None: + store_reference(self, index, cobj._objects) + arg = cobj._get_buffer_value() if self._type_._fficompositesize is None: self._buffer[index] = arg # something more sophisticated, cannot set field directly @@ -183,7 +209,7 @@ return self._length_ def _get_buffer_for_param(self): - return CArgObject(self._buffer.byptr()) + return CArgObject(self, self._buffer.byptr()) def _get_buffer_value(self): return self._buffer.buffer Modified: pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/basics.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/basics.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/basics.py Sat Dec 11 15:10:15 2010 @@ -46,21 +46,6 @@ else: return self.from_param(as_parameter) - def _CData_input(self, value): - """Used when data enters into ctypes from user code. 'value' is - some user-specified Python object, which is converted into a _rawffi - array of length 1 containing the same value according to the - type 'self'. - """ - cobj = self.from_param(value) - return cobj, cobj._get_buffer_for_param() - - def _CData_value(self, value): - cobj = self.from_param(value) - # we don't care here if this stuff will live afterwards, as we're - # interested only in value anyway - return cobj._get_buffer_value() - def _CData_output(self, resbuffer, base=None, index=-1): #assert isinstance(resbuffer, _rawffi.ArrayInstance) """Used when data exits ctypes and goes into user code. @@ -92,13 +77,23 @@ """ simple wrapper around buffer, just for the case of freeing it afterwards """ - def __init__(self, buffer): + def __init__(self, obj, buffer): + self._obj = obj self._buffer = buffer def __del__(self): self._buffer.free() self._buffer = None + def __repr__(self): + return '' % (self._obj,) + + def __eq__(self, other): + return self._obj == other + + def __ne__(self, other): + return self._obj != other + class _CData(object): """ The most basic object for all ctypes types """ @@ -128,7 +123,7 @@ return buffer(self._buffer) def _get_b_base(self): - return self._objects + return self._base _b_base_ = property(_get_b_base) _b_needsfree_ = False Modified: pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/builtin.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/builtin.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/builtin.py Sat Dec 11 15:10:15 2010 @@ -11,7 +11,8 @@ def _string_at(addr, lgt): # address here can be almost anything import ctypes - arg = ctypes.c_void_p._CData_value(addr) + cobj = ctypes.c_void_p.from_param(addr) + arg = cobj._get_buffer_value() return _rawffi.charp2rawstring(arg, lgt) def set_conversion_mode(encoding, errors): @@ -22,7 +23,8 @@ def _wstring_at(addr, lgt): import ctypes - arg = ctypes.c_void_p._CData_value(addr) + cobj = ctypes.c_void_p.from_param(addr) + arg = cobj._get_buffer_value() # XXX purely applevel if lgt == -1: lgt = sys.maxint Modified: pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/function.py Sat Dec 11 15:10:15 2010 @@ -35,6 +35,7 @@ _argtypes_ = None _restype_ = None + _errcheck_ = None _flags_ = 0 _ffiargshape = 'P' _ffishape = 'P' @@ -53,7 +54,15 @@ return self._argtypes_ def _setargtypes(self, argtypes): self._ptr = None - self._argtypes_ = argtypes + if argtypes is None: + self._argtypes_ = None + else: + for i, argtype in enumerate(argtypes): + if not hasattr(argtype, 'from_param'): + raise TypeError( + "item %d in _argtypes_ has no from_param method" % ( + i + 1,)) + self._argtypes_ = argtypes argtypes = property(_getargtypes, _setargtypes) def _getrestype(self): @@ -72,9 +81,24 @@ del self._restype_ restype = property(_getrestype, _setrestype, _delrestype) + def _geterrcheck(self): + return getattr(self, '_errcheck_', None) + def _seterrcheck(self, errcheck): + if not callable(errcheck): + raise TypeError("The errcheck attribute must be callable") + self._errcheck_ = errcheck + def _delerrcheck(self): + try: + del self._errcheck_ + except AttributeError: + pass + errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck) + def _ffishapes(self, args, restype): argtypes = [arg._ffiargshape for arg in args] if restype is not None: + if not isinstance(restype, _CDataMeta): + raise TypeError("invalid result type for callback function") restype = restype._ffiargshape else: restype = 'O' # void @@ -140,6 +164,7 @@ def __call__(self, *args): if self.callable is not None: + args = args[:len(self._argtypes_)] try: res = self.callable(*args) except: @@ -162,13 +187,29 @@ thisarg = None if argtypes is None: - argtypes = self._guess_argtypes(args) - argtypes, argsandobjs = self._wrap_args(argtypes, args) + argtypes = [] + args = self._convert_args(argtypes, args) + argtypes = [type(arg) for arg in args] restype = self._restype_ funcptr = self._getfuncptr(argtypes, restype, thisarg) - resbuffer = funcptr(*[arg._buffer for _, arg in argsandobjs]) - return self._build_result(restype, resbuffer, argtypes, argsandobjs) + resbuffer = funcptr(*[arg._get_buffer_for_param()._buffer + for arg in args]) + result = self._build_result(restype, resbuffer, argtypes, args) + + # The 'errcheck' protocol + if self._errcheck_: + v = self._errcheck_(result, self, args) + # If the errcheck funtion failed, let it throw + # If the errcheck function returned callargs unchanged, + # continue normal processing. + # If the errcheck function returned something else, + # use that as result. + if v is not args: + result = v + + return result + def _getfuncptr(self, argtypes, restype, thisarg=None): if self._ptr is not None and argtypes is self._argtypes_: @@ -212,31 +253,33 @@ raise @staticmethod - def _guess_argtypes(args): + def _conv_param(argtype, arg, index): from ctypes import c_char_p, c_wchar_p, c_void_p, c_int - res = [] - for arg in args: - if hasattr(arg, '_as_parameter_'): - arg = arg._as_parameter_ - if isinstance(arg, str): - res.append(c_char_p) - elif isinstance(arg, unicode): - res.append(c_wchar_p) - elif isinstance(arg, _CData): - res.append(type(arg)) - elif arg is None: - res.append(c_void_p) - #elif arg == 0: - # res.append(c_void_p) - elif isinstance(arg, (int, long)): - res.append(c_int) - else: - raise TypeError("Don't know how to handle %s" % (arg,)) - return res + if argtype is not None: + arg = argtype.from_param(arg) + if hasattr(arg, '_as_parameter_'): + arg = arg._as_parameter_ + + if isinstance(arg, _CData): + # The usual case when argtype is defined + cobj = arg + elif isinstance(arg, str): + cobj = c_char_p(arg) + elif isinstance(arg, unicode): + cobj = c_wchar_p(arg) + elif arg is None: + cobj = c_void_p() + elif isinstance(arg, (int, long)): + cobj = c_int(arg) + else: + raise TypeError("Don't know how to handle %s" % (arg,)) - def _wrap_args(self, argtypes, args): + return cobj + + def _convert_args(self, argtypes, args): wrapped_args = [] consumed = 0 + for i, argtype in enumerate(argtypes): defaultvalue = None if i > 0 and self._paramflags is not None: @@ -263,7 +306,7 @@ val = defaultvalue if val is None: val = 0 - wrapped = argtype._CData_input(val) + wrapped = self._conv_param(argtype, val, consumed) wrapped_args.append(wrapped) continue else: @@ -278,23 +321,22 @@ raise TypeError("Not enough arguments") try: - wrapped = argtype._CData_input(arg) - except (UnicodeError, TypeError), e: + wrapped = self._conv_param(argtype, arg, consumed) + except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) wrapped_args.append(wrapped) consumed += 1 if len(wrapped_args) < len(args): extra = args[len(wrapped_args):] - extra_types = self._guess_argtypes(extra) - for arg, argtype in zip(extra, extra_types): + argtypes = list(argtypes) + for i, arg in enumerate(extra): try: - wrapped = argtype._CData_input(arg) - except (UnicodeError, TypeError), e: + wrapped = self._conv_param(None, arg, i) + except (UnicodeError, TypeError, ValueError), e: raise ArgumentError(str(e)) wrapped_args.append(wrapped) - argtypes = list(argtypes) + extra_types - return argtypes, wrapped_args + return wrapped_args def _build_result(self, restype, resbuffer, argtypes, argsandobjs): """Build the function result: @@ -307,7 +349,7 @@ if self._com_index: if resbuffer[0] & 0x80000000: raise get_com_error(resbuffer[0], - self._com_iid, argsandobjs[0][0]) + self._com_iid, argsandobjs[0]) else: retval = int(resbuffer[0]) elif restype is not None: @@ -326,8 +368,8 @@ results = [] if self._paramflags: - for argtype, (obj, _), paramflag in zip(argtypes[1:], argsandobjs[1:], - self._paramflags): + for argtype, obj, paramflag in zip(argtypes[1:], argsandobjs[1:], + self._paramflags): if len(paramflag) == 2: idlflag, name = paramflag elif len(paramflag) == 3: Modified: pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/pointer.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/pointer.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/pointer.py Sat Dec 11 15:10:15 2010 @@ -1,7 +1,8 @@ import _rawffi from _ctypes.basics import _CData, _CDataMeta, cdata_from_address -from _ctypes.basics import sizeof, byref, keepalive_key +from _ctypes.basics import keepalive_key, store_reference, ensure_objects +from _ctypes.basics import sizeof, byref from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\ array_slice_setitem @@ -55,7 +56,8 @@ def set_type(self, TP): ffiarray = _rawffi.Array('P') def __init__(self, value=None): - self._buffer = ffiarray(1, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = ffiarray(1, autofree=True) if value is not None: self.contents = value self._ffiarray = ffiarray @@ -99,7 +101,10 @@ return self._type_._CData_output(self._subarray(index), self, index) def __setitem__(self, index, value): - self._subarray(index)[0] = self._type_._CData_value(value) + cobj = self._type_.from_param(value) + if ensure_objects(cobj) is not None: + store_reference(self, index, cobj._objects) + self._subarray(index)[0] = cobj._get_buffer_value() def __nonzero__(self): return self._buffer[0] != 0 @@ -110,29 +115,32 @@ if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()): raise TypeError("cast() argument 2 must be a pointer type, not %s" % (tp,)) - if isinstance(obj, Array): - ptr = tp.__new__(tp) - ptr._buffer = tp._ffiarray(1, autofree=True) - ptr._buffer[0] = obj._buffer - return ptr if isinstance(obj, (int, long)): result = tp() result._buffer[0] = obj return result - if obj is None: + elif obj is None: result = tp() return result - if not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): + elif isinstance(obj, Array): + ptr = tp.__new__(tp) + ptr._buffer = tp._ffiarray(1, autofree=True) + ptr._buffer[0] = obj._buffer + result = ptr + elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()): raise TypeError("cast() argument 1 must be a pointer, not %s" % (type(obj),)) - result = tp() + else: + result = tp() + result._buffer[0] = obj._buffer[0] # The casted objects '_objects' member: - # It must certainly contain the source objects one. + # From now on, both objects will use the same dictionary + # It must certainly contain the source objects # It must contain the source object itself. if obj._ensure_objects() is not None: - result._objects = {keepalive_key(0): obj._objects, - keepalive_key(1): obj} + result._objects = obj._objects + if isinstance(obj._objects, dict): + result._objects[id(obj)] = obj - result._buffer[0] = obj._buffer[0] return result Modified: pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/primitive.py Sat Dec 11 15:10:15 2010 @@ -132,8 +132,8 @@ ConvMode.errors) #self._objects = value array = _rawffi.Array('c')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -155,8 +155,8 @@ ConvMode.errors) #self._objects = value array = _rawffi.Array('u')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -174,8 +174,8 @@ def _setvalue(self, value): if isinstance(value, str): array = _rawffi.Array('c')(len(value)+1, value) + self._objects = CArgObject(value, array) value = array.buffer - self._objects = {'0': CArgObject(array)} elif value is None: value = 0 self._buffer[0] = value @@ -271,7 +271,9 @@ def _CData_output(self, resbuffer, base=None, index=-1): output = super(SimpleType, self)._CData_output(resbuffer, base, index) - return output.value + if self.__bases__[0] is _SimpleCData: + return output.value + return output def _sizeofinstances(self): return _rawffi.sizeof(self._type_) @@ -287,14 +289,15 @@ _type_ = 'i' def __init__(self, value=DEFAULT_VALUE): - self._buffer = self._ffiarray(1, autofree=True) + if not hasattr(self, '_buffer'): + self._buffer = self._ffiarray(1, autofree=True) if value is not DEFAULT_VALUE: self.value = value def _ensure_objects(self): - if self._type_ in 'zZ': - return self._objects - return None + if self._type_ not in 'zZP': + assert self._objects is None + return self._objects def _getvalue(self): return self._buffer[0] @@ -312,7 +315,11 @@ return self.value def __repr__(self): - return "%s(%r)" % (type(self).__name__, self.value) + if type(self).__bases__[0] is _SimpleCData: + return "%s(%r)" % (type(self).__name__, self.value) + else: + return "<%s object at 0x%x>" % (type(self).__name__, + id(self)) def __nonzero__(self): return self._buffer[0] not in (0, '\x00') Modified: pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/structure.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/structure.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_ctypes/structure.py Sat Dec 11 15:10:15 2010 @@ -34,9 +34,11 @@ if not isinstance(tp, _CDataMeta): raise TypeError("Expected CData subclass, got %s" % (tp,)) import ctypes - all_fields = _fields_[:] - for cls in inspect.getmro(superclass): - all_fields += getattr(cls, '_fields_', []) + all_fields = [] + for cls in reversed(inspect.getmro(superclass)): + # The first field comes from the most base class + all_fields.extend(getattr(cls, '_fields_', [])) + all_fields.extend(_fields_) names = [name for name, ctype in all_fields] rawfields = [(name, ctype._ffishape) for name, ctype in all_fields] @@ -168,7 +170,7 @@ def __init__(self, *args, **kwds): if len(args) > len(self._names): - raise TypeError("too many arguments") + raise TypeError("too many initializers") for name, arg in zip(self._names, args): if name in kwds: raise TypeError("duplicate value for argument %r" % ( Modified: pypy/branch/jit-unroll-loops/lib_pypy/_hashlib.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_hashlib.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_hashlib.py Sat Dec 11 15:10:15 2010 @@ -148,21 +148,29 @@ return hash(ctx, name) # shortcut functions +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + at builtinify def openssl_md5(string=''): return new('md5', string) + at builtinify def openssl_sha1(string=''): return new('sha1', string) + at builtinify def openssl_sha224(string=''): return new('sha224', string) + at builtinify def openssl_sha256(string=''): return new('sha256', string) + at builtinify def openssl_sha384(string=''): return new('sha384', string) + at builtinify def openssl_sha512(string=''): return new('sha512', string) - Modified: pypy/branch/jit-unroll-loops/lib_pypy/_locale.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_locale.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_locale.py Sat Dec 11 15:10:15 2010 @@ -11,6 +11,9 @@ # load the platform-specific cache made by running locale.ctc.py from ctypes_config_cache._locale_cache import * +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + # Ubuntu Gusty i386 structure class lconv(Structure): @@ -158,6 +161,7 @@ ul = ''.join(ul) string.letters = ul + at builtinify def setlocale(category, locale=None): "(integer,string=None) -> string. Activates/queries locale processing." if locale: @@ -182,6 +186,7 @@ groups.append(0) return groups + at builtinify def localeconv(): "() -> dict. Returns numeric and monetary locale-specific parameters." @@ -215,6 +220,7 @@ } return result + at builtinify def strcoll(s1, s2): "string,string -> int. Compares two strings according to the locale." @@ -233,6 +239,7 @@ # Collate the strings. return _wcscoll(s1, s2) + at builtinify def strxfrm(s): "string -> string. Returns a string that behaves for cmp locale-aware." @@ -246,6 +253,7 @@ _strxfrm(buf, s, n2) return buf.value + at builtinify def getdefaultlocale(): # TODO: Port code from CPython for Windows and Mac OS raise NotImplementedError() @@ -267,26 +275,31 @@ raise ValueError("unsupported langinfo constant") if HAS_LIBINTL: + @builtinify def gettext(msg): """gettext(msg) -> string Return translation of msg.""" return _gettext(msg) + @builtinify def dgettext(domain, msg): """dgettext(domain, msg) -> string Return translation of msg in domain.""" return _dgettext(domain, msg) + @builtinify def dcgettext(domain, msg, category): """dcgettext(domain, msg, category) -> string Return translation of msg in domain and category.""" return _dcgettext(domain, msg, category) + @builtinify def textdomain(domain): """textdomain(domain) -> string Set the C library's textdomain to domain, returning the new domain.""" return _textdomain(domain) + @builtinify def bindtextdomain(domain, dir): """bindtextdomain(domain, dir) -> string Bind the C library's domain to dir.""" @@ -297,6 +310,7 @@ return dirname if HAS_BIND_TEXTDOMAIN_CODESET: + @builtinify def bind_textdomain_codeset(domain, codeset): """bind_textdomain_codeset(domain, codeset) -> string Bind the C library's domain to codeset.""" Modified: pypy/branch/jit-unroll-loops/lib_pypy/_marshal.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_marshal.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_marshal.py Sat Dec 11 15:10:15 2010 @@ -6,6 +6,10 @@ import types from _codecs import utf_8_decode, utf_8_encode +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + TYPE_NULL = '0' TYPE_NONE = 'N' TYPE_FALSE = 'F' @@ -645,15 +649,18 @@ version = 1 + at builtinify def dump(x, f, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format m = _Marshaller(f.write) m.dump(x) + at builtinify def load(f): um = _Unmarshaller(f.read) return um.load() + at builtinify def dumps(x, version=version): # XXX 'version' is ignored, we always dump in a version-0-compatible format buffer = [] @@ -661,6 +668,7 @@ m.dump(x) return ''.join(buffer) + at builtinify def loads(s): um = _FastUnmarshaller(s) return um.load() Modified: pypy/branch/jit-unroll-loops/lib_pypy/_minimal_curses.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_minimal_curses.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_minimal_curses.py Sat Dec 11 15:10:15 2010 @@ -35,18 +35,24 @@ # ____________________________________________________________ +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + at builtinify def setupterm(termstr, fd): err = ctypes.c_int(0) result = clib.setupterm(termstr, fd, ctypes.byref(err)) if result == ERR: raise error("setupterm() failed (err=%d)" % err.value) + at builtinify def tigetstr(cap): result = clib.tigetstr(cap) if ctypes.cast(result, ctypes.c_void_p).value == ERR: return None return ctypes.cast(result, ctypes.c_char_p).value + at builtinify def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) if result is None: Modified: pypy/branch/jit-unroll-loops/lib_pypy/_pypy_interact.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/_pypy_interact.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/_pypy_interact.py Sat Dec 11 15:10:15 2010 @@ -4,6 +4,13 @@ def interactive_console(mainmodule=None): + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, 'ps1'): + sys.ps1 = '>>>> ' + if not hasattr(sys, 'ps2'): + sys.ps2 = '.... ' + # try: from _pypy_irc_topic import some_topic text = "And now for something completely different: ``%s''" % ( @@ -15,6 +22,7 @@ print text except ImportError: pass + # try: from pyrepl.simple_interact import check if not check(): Modified: pypy/branch/jit-unroll-loops/lib_pypy/binascii.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/binascii.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/binascii.py Sat Dec 11 15:10:15 2010 @@ -1,3 +1,9 @@ +"""A pure Python implementation of binascii. + +Rather slow and buggy in corner cases. +PyPy provides an RPython version too. +""" + class Error(Exception): pass @@ -277,7 +283,7 @@ if (c > '~' or c == '=' or (header and c == '_') or - (c == '.' and linelen == 0 and (inp == len(data) or + (c == '.' and linelen == 0 and (inp+1 == len(data) or data[inp+1] == '\n' or data[inp+1] == '\r')) or (not istext and (c == '\r' or c == '\n')) or Modified: pypy/branch/jit-unroll-loops/lib_pypy/cPickle.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/cPickle.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/cPickle.py Sat Dec 11 15:10:15 2010 @@ -5,6 +5,10 @@ from pickle import * from pickle import __doc__, __version__, format_version, compatible_formats +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + BadPickleGet = KeyError UnpickleableError = PicklingError @@ -31,9 +35,11 @@ def getvalue(self): return self.__f and self.__f.getvalue() + at builtinify def dump(obj, file, protocol=None): Pickler(file, protocol).dump(obj) + at builtinify def dumps(obj, protocol=None): file = StringIO() Pickler(file, protocol).dump(obj) Modified: pypy/branch/jit-unroll-loops/lib_pypy/cmath.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/cmath.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/cmath.py Sat Dec 11 15:10:15 2010 @@ -7,7 +7,10 @@ import math from math import e, pi - + +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + # constants _one = complex(1., 0.) @@ -24,6 +27,7 @@ return complex(real, imag) + at builtinify def acos(x): """acos(x) @@ -32,6 +36,7 @@ return -(_prodi(log((x+(_i*sqrt((_one-(x*x)))))))) + at builtinify def acosh(x): """acosh(x) @@ -41,6 +46,7 @@ return z+z + at builtinify def asin(x): """asin(x) @@ -52,6 +58,7 @@ return -(_prodi(log((sqrt_1_minus_x_sq+_prodi(x))))) + at builtinify def asinh(x): """asinh(x) @@ -61,6 +68,7 @@ return z+z + at builtinify def atan(x): """atan(x) @@ -69,6 +77,7 @@ return _halfi*log(((_i+x)/(_i-x))) + at builtinify def atanh(x): """atanh(x) @@ -77,6 +86,7 @@ return _half*log((_one+x)/(_one-x)) + at builtinify def cos(x): """cos(x) @@ -88,6 +98,7 @@ return complex(real, imag) + at builtinify def cosh(x): """cosh(x) @@ -99,6 +110,7 @@ return complex(real, imag) + at builtinify def exp(x): """exp(x) @@ -111,6 +123,7 @@ return complex(real, imag) + at builtinify def log(x, base=None): """log(x) @@ -125,6 +138,7 @@ return complex(real, imag) + at builtinify def log10(x): """log10(x) @@ -137,6 +151,7 @@ return complex(real, imag) + at builtinify def sin(x): """sin(x) @@ -148,6 +163,7 @@ return complex(real, imag) + at builtinify def sinh(x): """sinh(x) @@ -159,6 +175,7 @@ return complex(real, imag) + at builtinify def sqrt(x): """sqrt(x) @@ -184,6 +201,7 @@ _sqrt_half = sqrt(_half) + at builtinify def tan(x): """tan(x) @@ -204,6 +222,7 @@ return complex(real, imag) + at builtinify def tanh(x): """tanh(x) Modified: pypy/branch/jit-unroll-loops/lib_pypy/ctypes_support.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/ctypes_support.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/ctypes_support.py Sat Dec 11 15:10:15 2010 @@ -25,7 +25,7 @@ def _where_is_errno(): return standard_c_lib.__errno_location() -elif sys.platform == 'darwin': +elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() Modified: pypy/branch/jit-unroll-loops/lib_pypy/grp.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/grp.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/grp.py Sat Dec 11 15:10:15 2010 @@ -9,6 +9,10 @@ from ctypes import Structure, c_char_p, c_int, POINTER from ctypes_support import standard_c_lib as libc +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + gid_t = c_int class GroupStruct(Structure): @@ -64,6 +68,7 @@ return Group(res.contents.gr_name, res.contents.gr_passwd, res.contents.gr_gid, mem) + at builtinify def getgrgid(gid): res = libc.getgrgid(gid) if not res: @@ -71,6 +76,7 @@ raise KeyError(gid) return _group_from_gstruct(res) + at builtinify def getgrnam(name): if not isinstance(name, str): raise TypeError("expected string") @@ -79,6 +85,7 @@ raise KeyError(name) return _group_from_gstruct(res) + at builtinify def getgrall(): libc.setgrent() lst = [] Modified: pypy/branch/jit-unroll-loops/lib_pypy/hashlib.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/hashlib.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/hashlib.py Sat Dec 11 15:10:15 2010 @@ -50,53 +50,30 @@ 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' """ +import sys try: import _hashlib except ImportError: _hashlib = None -def __get_builtin_constructor(name): - if name in ('SHA1', 'sha1'): - import sha - return sha.new - elif name in ('MD5', 'md5'): - import md5 - return md5.new - elif name in ('SHA256', 'sha256'): - import _sha256 - return _sha256.sha256 - elif name in ('SHA224', 'sha224'): - import _sha256 - return _sha256.sha224 - elif name in ('SHA512', 'sha512'): - import _sha512 - return _sha512.sha512 - elif name in ('SHA384', 'sha384'): - import _sha512 - return _sha512.sha384 - raise ValueError, "unsupported hash type" - def __hash_new(name, string=''): """new(name, string='') - Return a new hashing object using the named algorithm; optionally initialized with a string. """ try: - if _hashlib: - return _hashlib.new(name, string) - except ValueError: - # If the _hashlib module (OpenSSL) doesn't support the named - # hash, try using our builtin implementations. - # This allows for SHA224/256 and SHA384/512 support even though - # the OpenSSL library prior to 0.9.8 doesn't provide them. - pass - - return __get_builtin_constructor(name)(string) + new = __byname[name] + except KeyError: + raise ValueError("unsupported hash type") + return new(string) new = __hash_new -def _setfuncs(): - # use the wrapper of the C implementation +# ____________________________________________________________ + +__byname = {} +def __use_openssl_funcs(): + # use the wrapper of the C implementation sslprefix = 'openssl_' for opensslfuncname, func in vars(_hashlib).items(): if not opensslfuncname.startswith(sslprefix): @@ -106,23 +83,41 @@ # try them all, some may not work due to the OpenSSL # version not supporting that algorithm. func() - # Use the C function directly (very fast) - globals()[funcname] = func + # Use the C function directly (very fast, but with ctypes overhead) + __byname[funcname] = func except ValueError: - try: - # Use the builtin implementation directly (fast) - globals()[funcname] = __get_builtin_constructor(funcname) - except ValueError: - # this one has no builtin implementation, don't define it - pass + pass + +def __use_builtin_funcs(): + # look up the built-in versions (written in Python or RPython), + # and use the fastest one: + # 1. the one in RPython + # 2. the one from openssl (slower due to ctypes calling overhead) + # 3. the one in pure Python + if 'sha1' not in __byname or 'sha' in sys.builtin_module_names: + import sha + __byname['sha1'] = sha.new + if 'md5' not in __byname or 'md5' in sys.builtin_module_names: + import md5 + __byname['md5'] = md5.new + if 'sha256' not in __byname: + import _sha256 + __byname['sha256'] = _sha256.sha256 + if 'sha224' not in __byname: + import _sha256 + __byname['sha224'] = _sha256.sha224 + if 'sha512' not in __byname: + import _sha512 + __byname['sha512'] = _sha512.sha512 + if 'sha384' not in __byname: + import _sha512 + __byname['sha384'] = _sha512.sha384 + +def __export_funcs(): + for key, value in __byname.items(): + globals()[key] = __byname[key.upper()] = value if _hashlib: - _setfuncs() -else: - # lookup the C function to use directly for the named constructors - md5 = __get_builtin_constructor('md5') - sha1 = __get_builtin_constructor('sha1') - sha224 = __get_builtin_constructor('sha224') - sha256 = __get_builtin_constructor('sha256') - sha384 = __get_builtin_constructor('sha384') - sha512 = __get_builtin_constructor('sha512') + __use_openssl_funcs() +__use_builtin_funcs() +__export_funcs() Modified: pypy/branch/jit-unroll-loops/lib_pypy/itertools.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/itertools.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/itertools.py Sat Dec 11 15:10:15 2010 @@ -27,6 +27,9 @@ 'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap', 'takewhile', 'tee'] +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + class chain(object): """Make an iterator that returns elements from the first iterable @@ -565,7 +568,8 @@ def __iter__(self): return self - + + at builtinify def tee(iterable, n=2): """Return n independent iterators from a single iterable. Note : once tee() has made a split, the original iterable Modified: pypy/branch/jit-unroll-loops/lib_pypy/msvcrt.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/msvcrt.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/msvcrt.py Sat Dec 11 15:10:15 2010 @@ -17,6 +17,10 @@ except AttributeError: # we are not on windows raise ImportError +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + open_osfhandle.argtypes = [ctypes.c_int, ctypes.c_int] open_osfhandle.restype = ctypes.c_int @@ -34,6 +38,7 @@ _locking.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] _locking.restype = ctypes.c_int + at builtinify def locking(fd, mode, nbytes): '''lock or unlock a number of bytes in a file.''' rv = _locking(fd, mode, nbytes) Modified: pypy/branch/jit-unroll-loops/lib_pypy/pwd.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/pwd.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/pwd.py Sat Dec 11 15:10:15 2010 @@ -17,6 +17,10 @@ from ctypes_support import standard_c_lib as libc from ctypes import Structure, POINTER, c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + uid_t = c_int gid_t = c_int @@ -79,10 +83,12 @@ _endpwent.argtypes = None _endpwent.restype = None + at builtinify def mkpwent(pw): pw = pw.contents return struct_passwd(pw) + at builtinify def getpwuid(uid): """ getpwuid(uid) -> (pw_name,pw_passwd,pw_uid, @@ -95,6 +101,7 @@ raise KeyError("getpwuid(): uid not found: %s" % uid) return mkpwent(pw) + at builtinify def getpwnam(name): """ getpwnam(name) -> (pw_name,pw_passwd,pw_uid, @@ -109,9 +116,10 @@ raise KeyError("getpwname(): name not found: %s" % name) return mkpwent(pw) + at builtinify def getpwall(): """ - "getpwall() -> list_of_entries + getpwall() -> list_of_entries Return a list of all available password database entries, in arbitrary order. See pwd.__doc__ for more on password database entries. """ Modified: pypy/branch/jit-unroll-loops/lib_pypy/pyexpat.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/pyexpat.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/pyexpat.py Sat Dec 11 15:10:15 2010 @@ -7,6 +7,9 @@ # load the platform-specific cache made by running pyexpat.ctc.py from ctypes_config_cache._pyexpat_cache import * +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + lib = ctypes.CDLL(ctypes.util.find_library('expat')) @@ -425,9 +428,11 @@ new_parser._set_unknown_encoding_handler() return new_parser + at builtinify def ErrorString(errno): return XML_ErrorString(errno)[:200] + at builtinify def ParserCreate(encoding=None, namespace_separator=None, intern=None): if (not isinstance(encoding, str) and not encoding is None): Modified: pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/test_hashlib.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/test_hashlib.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/test_hashlib.py Sat Dec 11 15:10:15 2010 @@ -2,10 +2,20 @@ from ..ctypes_config_cache import rebuild rebuild.rebuild_one('hashlib.ctc.py') +from . import hack___pypy__ from .. import hashlib, _hashlib def test_unicode(): - assert isinstance(hashlib.new('sha1', u'xxx'), _hashlib.hash) + assert isinstance(hashlib.new('sha256', u'xxx'), _hashlib.hash) + +pure_python_version = { + 'md5': 'md5.new', + 'sha1': 'sha.new', + 'sha224': '_sha256.sha224', + 'sha256': '_sha256.sha256', + 'sha384': '_sha512.sha384', + 'sha512': '_sha512.sha512', + } def test_attributes(): for name, expected_size in {'md5': 16, @@ -30,7 +40,10 @@ assert hexdigest == h.hexdigest() # also test the pure Python implementation - h = hashlib.__get_builtin_constructor(name)('') + modname, constructor = pure_python_version[name].split('.') + mod = __import__('lib_pypy.' + modname, None, None, ['__doc__']) + builder = getattr(mod, constructor) + h = builder('') assert h.digest_size == expected_size assert h.digestsize == expected_size # Modified: pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/test_structseq.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/test_structseq.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/pypy_test/test_structseq.py Sat Dec 11 15:10:15 2010 @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from .._structseq import * +from .._structseq import structseqfield, structseqtype class mydata: Modified: pypy/branch/jit-unroll-loops/lib_pypy/readline.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/readline.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/readline.py Sat Dec 11 15:10:15 2010 @@ -6,8 +6,4 @@ are only stubs at the moment. """ -# Note that PyPy contains also a built-in module 'readline' which will hide -# this one if compiled in. However the built-in module is incomplete; -# don't use it. - from pyrepl.readline import * Modified: pypy/branch/jit-unroll-loops/lib_pypy/resource.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/resource.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/resource.py Sat Dec 11 15:10:15 2010 @@ -11,6 +11,10 @@ from errno import EINVAL, EPERM import _structseq +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + class error(Exception): pass @@ -77,6 +81,7 @@ ru_nvcsw = _structseq.structseqfield(14) ru_nivcsw = _structseq.structseqfield(15) + at builtinify def rlimit_check_bounds(rlim_cur, rlim_max): if rlim_cur > rlim_t_max: raise ValueError("%d does not fit into rlim_t" % rlim_cur) @@ -89,6 +94,7 @@ ("rlim_max", rlim_t), ) + at builtinify def getrusage(who): ru = _struct_rusage() ret = _getrusage(who, byref(ru)) @@ -116,6 +122,7 @@ ru.ru_nivcsw, )) + at builtinify def getrlimit(resource): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -127,6 +134,7 @@ raise error(errno) return (rlim.rlim_cur, rlim.rlim_max) + at builtinify def setrlimit(resource, rlim): if not(0 <= resource < RLIM_NLIMITS): return ValueError("invalid resource specified") @@ -143,6 +151,7 @@ else: raise error(errno) + at builtinify def getpagesize(): pagesize = 0 if _getpagesize: Modified: pypy/branch/jit-unroll-loops/lib_pypy/syslog.py ============================================================================== --- pypy/branch/jit-unroll-loops/lib_pypy/syslog.py (original) +++ pypy/branch/jit-unroll-loops/lib_pypy/syslog.py Sat Dec 11 15:10:15 2010 @@ -15,6 +15,10 @@ from ctypes_support import standard_c_lib as libc from ctypes import c_int, c_char_p +try: from __pypy__ import builtinify +except ImportError: builtinify = lambda f: f + + # Real prototype is: # void syslog(int priority, const char *format, ...); # But we also need format ("%s") and one format argument (message) @@ -34,9 +38,11 @@ _setlogmask.argtypes = (c_int,) _setlogmask.restype = c_int + at builtinify def openlog(ident, option, facility): _openlog(ident, option, facility) + at builtinify def syslog(arg1, arg2=None): if arg2 is not None: priority, message = arg1, arg2 @@ -44,15 +50,19 @@ priority, message = LOG_INFO, arg1 _syslog(priority, "%s", message) + at builtinify def closelog(): _closelog() + at builtinify def setlogmask(mask): return _setlogmask(mask) + at builtinify def LOG_MASK(pri): return (1 << pri) + at builtinify def LOG_UPTO(pri): return (1 << (pri + 1)) - 1 Modified: pypy/branch/jit-unroll-loops/pypy/annotation/annrpython.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/annrpython.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/annrpython.py Sat Dec 11 15:10:15 2010 @@ -145,7 +145,7 @@ classdef.add_source_for_attribute(attr, classdef.classdesc) self.bookkeeper assert isinstance(s_result, annmodel.SomePBC) - olddesc = s_result.descriptions.iterkeys().next() + olddesc = s_result.any_description() desc = olddesc.bind_self(classdef) args = self.bookkeeper.build_args("simple_call", args_s[:]) desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc], Modified: pypy/branch/jit-unroll-loops/pypy/annotation/binaryop.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/binaryop.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/binaryop.py Sat Dec 11 15:10:15 2010 @@ -861,7 +861,7 @@ def getitem((p, obj)): assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - def setitem((p, obj)): + def setitem((p, obj), s_value): assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) class __extend__(pairtype(SomeObject, SomePtr)): Modified: pypy/branch/jit-unroll-loops/pypy/annotation/bookkeeper.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/bookkeeper.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/bookkeeper.py Sat Dec 11 15:10:15 2010 @@ -262,7 +262,7 @@ args_s, s_result) def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result): - descs = s_callable.descriptions.keys() + descs = list(s_callable.descriptions) if not descs: return family = descs[0].getcallfamily() @@ -590,7 +590,7 @@ assert s_attr.is_constant() attr = s_attr.const - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue @@ -633,7 +633,7 @@ """Analyse a call to a SomePBC() with the given args (list of annotations). """ - descs = pbc.descriptions.keys() + descs = list(pbc.descriptions) if not descs: return s_ImpossibleValue first = descs[0] Modified: pypy/branch/jit-unroll-loops/pypy/annotation/description.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/description.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/description.py Sat Dec 11 15:10:15 2010 @@ -636,6 +636,24 @@ return self return None + def maybe_return_immutable_list(self, attr, s_result): + # hack: 'x.lst' where lst is listed in _immutable_fields_ as 'lst[*]' + # should really return an immutable list as a result. Implemented + # by changing the result's annotation (but not, of course, doing an + # actual copy in the rtyper). Tested in pypy.rpython.test.test_rlist, + # test_immutable_list_out_of_instance. + search = '%s[*]' % (attr,) + cdesc = self + while cdesc is not None: + if '_immutable_fields_' in cdesc.classdict: + if search in cdesc.classdict['_immutable_fields_'].value: + s_result.listdef.never_resize() + s_copy = s_result.listdef.offspring() + s_copy.listdef.mark_as_immutable() + return s_copy + cdesc = cdesc.basedesc + return s_result # common case + def consider_call_site(bookkeeper, family, descs, args, s_result): from pypy.annotation.model import SomeInstance, SomePBC, s_None if len(descs) == 1: @@ -654,7 +672,7 @@ if isinstance(s_init, SomePBC): assert len(s_init.descriptions) == 1, ( "unexpected dynamic __init__?") - initfuncdesc = s_init.descriptions.keys()[0] + initfuncdesc, = s_init.descriptions if isinstance(initfuncdesc, FunctionDesc): initmethdesc = bookkeeper.getmethoddesc(initfuncdesc, classdef, @@ -782,8 +800,8 @@ desc.selfclassdef, desc.name, commonflags) - del descs[desc] - descs[newdesc] = None + descs.remove(desc) + descs.add(newdesc) # --- case 1 --- groups = {} @@ -798,7 +816,7 @@ for desc2 in group: cdef2 = desc2.selfclassdef if cdef1 is not cdef2 and cdef1.issubclass(cdef2): - del descs[desc1] + descs.remove(desc1) break simplify_desc_set = staticmethod(simplify_desc_set) Modified: pypy/branch/jit-unroll-loops/pypy/annotation/listdef.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/listdef.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/listdef.py Sat Dec 11 15:10:15 2010 @@ -6,11 +6,16 @@ class TooLateForChange(Exception): pass +class ListChangeUnallowed(Exception): + pass + class ListItem(object): mutated = False # True for lists mutated after creation resized = False # True for lists resized after creation range_step = None # the step -- only for lists only created by a range() dont_change_any_more = False # set to True when too late for changes + immutable = False # for getattr out of _immutable_fields_ = ['attr[*]'] + must_not_resize = False # make_sure_not_resized() # what to do if range_step is different in merge. # - if one is a list (range_step is None), unify to a list. @@ -26,7 +31,6 @@ self.bookkeeper = bookkeeper self.itemof = {} # set of all ListDefs using this ListItem self.read_locations = {} - self.dont_resize = False if bookkeeper is None: self.dont_change_any_more = True @@ -34,12 +38,15 @@ if not self.mutated: if self.dont_change_any_more: raise TooLateForChange + self.immutable = False self.mutated = True def resize(self): if not self.resized: - if self.dont_change_any_more or self.dont_resize: + if self.dont_change_any_more: raise TooLateForChange + if self.must_not_resize: + raise ListChangeUnallowed("resizing list") self.resized = True def setrangestep(self, step): @@ -63,11 +70,13 @@ # things more general self, other = other, self - if other.dont_resize: - if self.resized: - raise TooLateForChange() - self.dont_resize = True - if other.mutated: self.mutate() + self.immutable &= other.immutable + if other.must_not_resize: + if self.resized: + raise ListChangeUnallowed("list merge with a resized") + self.must_not_resize = True + if other.mutated: + self.mutate() if other.resized: self.resize() if other.range_step != self.range_step: @@ -176,9 +185,11 @@ self.listitem.generalize(s_value) def __repr__(self): - return '<[%r]%s%s>' % (self.listitem.s_value, + return '<[%r]%s%s%s%s>' % (self.listitem.s_value, self.listitem.mutated and 'm' or '', - self.listitem.resized and 'r' or '') + self.listitem.resized and 'r' or '', + self.listitem.immutable and 'I' or '', + self.listitem.must_not_resize and '!R' or '') def mutate(self): self.listitem.mutate() @@ -189,13 +200,20 @@ def never_resize(self): if self.listitem.resized: - raise TooLateForChange() - self.listitem.dont_resize = True + raise ListChangeUnallowed("list already resized") + self.listitem.must_not_resize = True - def never_mutate(self): - if self.listitem.resized or self.listitem.mutated: - raise TooLateForChange() - self.listitem.dont_change_any_more = True + def mark_as_immutable(self): + # Sets the 'immutable' flag. Note that unlike "never resized", + # the immutable flag is only a hint. It is cleared again e.g. + # when we merge with a "normal" list that doesn't have it. It + # is thus expected to live only shortly, mostly for the case + # of writing 'x.list[n]'. + self.never_resize() + if not self.listitem.mutated: + self.listitem.immutable = True + #else: it's fine, don't set immutable=True at all (see + # test_can_merge_immutable_list_with_regular_list) MOST_GENERAL_LISTDEF = ListDef(None, SomeObject()) Modified: pypy/branch/jit-unroll-loops/pypy/annotation/model.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/model.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/model.py Sat Dec 11 15:10:15 2010 @@ -356,8 +356,8 @@ immutable = True def __init__(self, descriptions, can_be_None=False, subset_of=None): - # descriptions is a set of Desc instances. - descriptions = dict.fromkeys(descriptions) + # descriptions is a set of Desc instances + descriptions = set(descriptions) self.descriptions = descriptions self.can_be_None = can_be_None self.subset_of = subset_of @@ -379,6 +379,9 @@ if desc.pyobj is not None: self.const = desc.pyobj + def any_description(self): + return iter(self.descriptions).next() + def getKind(self): "Return the common Desc class of all descriptions in this PBC." kinds = {} Modified: pypy/branch/jit-unroll-loops/pypy/annotation/specialize.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/specialize.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/specialize.py Sat Dec 11 15:10:15 2010 @@ -345,7 +345,8 @@ key.append(s.const) elif isinstance(s, SomePBC) and len(s.descriptions) == 1: # for test_specialize_arg_bound_method - key.append(s.descriptions.keys()[0]) + desc, = s.descriptions + key.append(desc) else: raise Exception("specialize:arg(%d): argument not constant: %r" % (i, s)) Modified: pypy/branch/jit-unroll-loops/pypy/annotation/test/test_annrpython.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/test/test_annrpython.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/test/test_annrpython.py Sat Dec 11 15:10:15 2010 @@ -10,7 +10,7 @@ from pypy.translator.translator import graphof as tgraphof from pypy.annotation import policy from pypy.annotation import specialize -from pypy.annotation.listdef import ListDef, TooLateForChange +from pypy.annotation.listdef import ListDef, ListChangeUnallowed from pypy.annotation.dictdef import DictDef from pypy.objspace.flow.model import * from pypy.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong @@ -1010,7 +1010,7 @@ bookkeeper = a.bookkeeper def getmdesc(bmeth): - return bookkeeper.immutablevalue(bmeth).descriptions.keys()[0] + return bookkeeper.immutablevalue(bmeth).any_description() mdescA_m = getmdesc(A().m) mdescC_m = getmdesc(C().m) @@ -2862,7 +2862,7 @@ assert s.items[0].flags == {'access_directly': True} assert isinstance(s.items[1], annmodel.SomePBC) assert len(s.items[1].descriptions) == 1 - assert s.items[1].descriptions.keys()[0].flags == {'access_directly': + assert s.items[1].any_description().flags == {'access_directly': True} assert isinstance(s.items[2], annmodel.SomeInstance) assert s.items[2].flags == {'access_directly': True} @@ -3206,7 +3206,7 @@ l.append(4) a = self.RPythonAnnotator() - py.test.raises(TooLateForChange, a.build_types, g, []) + py.test.raises(ListChangeUnallowed, a.build_types, g, []) assert called def test_listitem_no_mutating2(self): @@ -3229,7 +3229,7 @@ a = self.RPythonAnnotator() a.translator.config.translation.list_comprehension_operations = True - py.test.raises(TooLateForChange, a.build_types, fn, [int]) + py.test.raises(ListChangeUnallowed, a.build_types, fn, [int]) def test_listitem_never_resize(self): from pypy.rlib.debug import check_annotation @@ -3243,7 +3243,7 @@ check_annotation(l, checker) a = self.RPythonAnnotator() - py.test.raises(TooLateForChange, a.build_types, f, []) + py.test.raises(ListChangeUnallowed, a.build_types, f, []) def test_len_of_empty_list(self): @@ -3357,6 +3357,87 @@ # not a constant: both __enter__ and __exit__ have been annotated assert not s.is_constant() + def test_make_sure_not_resized(self): + from pypy.rlib.debug import make_sure_not_resized + + def pycode(consts): + make_sure_not_resized(consts) + def build1(): + return pycode(consts=[1]) + def build2(): + return pycode(consts=[0]) + def fn(): + build1() + build2() + + a = self.RPythonAnnotator() + a.translator.config.translation.list_comprehension_operations = True + a.build_types(fn, []) + # assert did not raise ListChangeUnallowed + + def test_return_immutable_list(self): + class A: + _immutable_fields_ = 'lst[*]' + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + return a.lst + + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert s.listdef.listitem.immutable + + def test_immutable_list_is_actually_resized(self): + class A: + _immutable_fields_ = 'lst[*]' + def f(n): + a = A() + l1 = [n] + l1.append(n+1) + a.lst = l1 + return a.lst + + a = self.RPythonAnnotator() + py.test.raises(ListChangeUnallowed, a.build_types, f, [int]) + + def test_can_merge_immutable_list_with_regular_list(self): + class A: + _immutable_fields_ = 'lst[*]' + def foo(lst): + pass + + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + if n > 0: + foo(a.lst) + else: + lst = [0] + lst[0] = n + foo(lst) + + a = self.RPythonAnnotator() + a.build_types(f, [int]) + + def f(n): + a = A() + l1 = [n, 0] + l1[1] = n+1 + a.lst = l1 + if n > 0: + lst = [0] + lst[0] = n + foo(lst) + else: + foo(a.lst) + + a = self.RPythonAnnotator() + a.build_types(f, [int]) + def g(n): return [0,1,2,n] Modified: pypy/branch/jit-unroll-loops/pypy/annotation/unaryop.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/annotation/unaryop.py (original) +++ pypy/branch/jit-unroll-loops/pypy/annotation/unaryop.py Sat Dec 11 15:10:15 2010 @@ -615,6 +615,9 @@ if basedef.classdesc.all_enforced_attrs is not None: if attr in basedef.classdesc.all_enforced_attrs: raise HarmlesslyBlocked("get enforced attr") + elif isinstance(s_result, SomeList): + s_result = ins.classdef.classdesc.maybe_return_immutable_list( + attr, s_result) return s_result return SomeObject() getattr.can_only_throw = [] Modified: pypy/branch/jit-unroll-loops/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/config/pypyoption.py (original) +++ pypy/branch/jit-unroll-loops/pypy/config/pypyoption.py Sat Dec 11 15:10:15 2010 @@ -31,7 +31,7 @@ "crypt", "signal", "_rawffi", "termios", "zlib", "struct", "md5", "sha", "bz2", "_minimal_curses", "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", - "_bisect"] + "_bisect", "binascii"] )) translation_modules = default_modules.copy() @@ -76,6 +76,7 @@ "_rawffi": [("objspace.usemodules.struct", True)], "cpyext": [("translation.secondaryentrypoints", "cpyext"), ("translation.shared", sys.platform == "win32")], + "_ffi": [("translation.jit_ffi", True)], } module_import_dependencies = { @@ -161,7 +162,6 @@ suggests=[("objspace.allworkingmodules", False)]), BoolOption("geninterp", "specify whether geninterp should be used", - cmdline=None, default=True), BoolOption("logbytecodes", Modified: pypy/branch/jit-unroll-loops/pypy/config/translationoption.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/config/translationoption.py (original) +++ pypy/branch/jit-unroll-loops/pypy/config/translationoption.py Sat Dec 11 15:10:15 2010 @@ -3,6 +3,7 @@ from pypy.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption, FloatOption from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config from pypy.config.config import ConfigError +from pypy.config.support import detect_number_of_processors DEFL_INLINE_THRESHOLD = 32.4 # just enough to inline add__Int_Int() # and just small enough to prevend inlining of some rlist functions. @@ -113,13 +114,10 @@ ChoiceOption("jit_backend", "choose the backend for the JIT", ["auto", "x86", "x86-without-sse2", "llvm"], default="auto", cmdline="--jit-backend"), - ChoiceOption("jit_debug", "the amount of debugging dumps made by the JIT", - ["off", "profile", "steps", "detailed"], - default="profile", # XXX for now - cmdline="--jit-debug"), ChoiceOption("jit_profiler", "integrate profiler support into the JIT", ["off", "oprofile"], default="off"), + BoolOption("jit_ffi", "optimize libffi calls", default=False), # misc BoolOption("verbose", "Print extra information", default=False), @@ -171,7 +169,7 @@ default=False, negation=False), IntOption("make_jobs", "Specify -j argument to make for compilation" " (C backend only)", - cmdline="--make-jobs", default=1), + cmdline="--make-jobs", default=detect_number_of_processors()), # Flags of the TranslationContext: BoolOption("simplifying", "Simplify flow graphs", default=True), Modified: pypy/branch/jit-unroll-loops/pypy/conftest.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/conftest.py (original) +++ pypy/branch/jit-unroll-loops/pypy/conftest.py Sat Dec 11 15:10:15 2010 @@ -336,13 +336,15 @@ self.runtest_finish() def runtest_open(self): - leakfinder.start_tracking_allocations() + if not getattr(self.obj, 'dont_track_allocations', False): + leakfinder.start_tracking_allocations() def runtest_perform(self): super(PyPyTestFunction, self).runtest() def runtest_close(self): - if leakfinder.TRACK_ALLOCATIONS: + if (not getattr(self.obj, 'dont_track_allocations', False) + and leakfinder.TRACK_ALLOCATIONS): self._pypytest_leaks = leakfinder.stop_tracking_allocations(False) else: # stop_tracking_allocations() already called self._pypytest_leaks = None Modified: pypy/branch/jit-unroll-loops/pypy/doc/cpython_differences.txt ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/doc/cpython_differences.txt (original) +++ pypy/branch/jit-unroll-loops/pypy/doc/cpython_differences.txt Sat Dec 11 15:10:15 2010 @@ -20,16 +20,21 @@ __builtin__ `__pypy__`_ + _ast + _bisect _codecs _lsprof `_minimal_curses`_ _random `_rawffi`_ + _ssl _socket _sre _weakref + array bz2 cStringIO + `cpyext`_ crypt errno exceptions @@ -72,7 +77,7 @@ * Supported by being rewritten in pure Python (possibly using ``ctypes``): see the `lib_pypy/`_ directory. Examples of modules that we - support this way: ``ctypes``, ``array``, ``cPickle``, + support this way: ``ctypes``, ``cPickle``, ``cStringIO``, ``cmath``, ``dbm`` (?), ``datetime``, ``binascii``... Note that some modules are both in there and in the list above; by default, the built-in module is used (but can be disabled @@ -80,11 +85,13 @@ The extension modules (i.e. modules written in C, in the standard CPython) that are neither mentioned above nor in `lib_pypy/`_ are not available in PyPy. +(You may have a chance to use them anyway with `cpyext`_.) .. the nonstandard modules are listed below... .. _`__pypy__`: __pypy__-module.html .. _`_rawffi`: ctypes-implementation.html .. _`_minimal_curses`: config/objspace.usemodules._minimal_curses.html +.. _`cpyext`: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html .. _Stackless: stackless.html @@ -129,12 +136,10 @@ .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html -The built-in function ``id()`` returns numbers that are not addresses -for most of PyPy's garbage collectors. -This is most visible in the default repr: a typical PyPy object can -pretend to be located ``at 0x00000009``. This is just its ``id()``, not -its real address (because an object can move around in some GCs). Calling -``id`` a lot can lead to performance problem. +Using the default GC called ``minimark``, the built-in function ``id()`` +works like it does in CPython. With other GCs it returns numbers that +are not real addresses (because an object can move around several times) +and calling it a lot can lead to performance problem. Note that if you have a long chain of objects, each with a reference to the next one, and each with a __del__, PyPy's GC will perform badly. On Modified: pypy/branch/jit-unroll-loops/pypy/doc/faq.txt ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/doc/faq.txt (original) +++ pypy/branch/jit-unroll-loops/pypy/doc/faq.txt Sat Dec 11 15:10:15 2010 @@ -47,7 +47,7 @@ There is also an experimental support for CPython extension modules, so they'll run without change (from current observation, rather with little -change) on trunk. It has been a part of 1.3 release, but support is still +change) on trunk. It has been a part of 1.4 release, but support is still in alpha phase. .. _`extension modules`: cpython_differences.html#extension-modules Modified: pypy/branch/jit-unroll-loops/pypy/doc/index.txt ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/doc/index.txt (original) +++ pypy/branch/jit-unroll-loops/pypy/doc/index.txt Sat Dec 11 15:10:15 2010 @@ -8,7 +8,7 @@ Getting into PyPy ... ============================================= -* `Release 1.3`_: the latest official release +* `Release 1.4`_: the latest official release * `PyPy Blog`_: news and status info about PyPy @@ -56,4 +56,4 @@ .. _`Documentation`: docindex.html .. _`Getting Started`: getting-started.html .. _papers: extradoc.html -.. _`Release 1.3`: http://pypy.org/download.html +.. _`Release 1.4`: http://pypy.org/download.html Modified: pypy/branch/jit-unroll-loops/pypy/doc/release-1.4.0.txt ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/doc/release-1.4.0.txt (original) +++ pypy/branch/jit-unroll-loops/pypy/doc/release-1.4.0.txt Sat Dec 11 15:10:15 2010 @@ -2,27 +2,58 @@ PyPy 1.4: Ouroboros in practice =============================== -Hello. - We're pleased to announce the 1.4 release of PyPy. This is a major breakthrough in our long journey, as PyPy 1.4 is the first PyPy release that can translate -itself faster than CPython. Starting today, we plan to start using PyPy for our -own development. +itself faster than CPython. Starting today, we are using PyPy more for +our every-day development. So may you :) You can download it here: + + http://pypy.org/download.html + +What is PyPy +============ + +PyPy is a very compliant Python interpreter, almost a drop-in replacement +for CPython. It's fast (`pypy 1.4 and cpython 2.6`_ comparison) -Among other features, this release includes numerous performance improvements -(which made fast self-hosting possible), a 64-bit JIT backend, as well as serious -stabilization. As of now, we can consider the 32-bit version of PyPy stable enough -to run in production. +Among its new features, this release includes numerous performance improvements +(which made fast self-hosting possible), a 64-bit JIT backend, as well +as serious stabilization. As of now, we can consider the 32-bit and 64-bit +linux versions of PyPy stable enough to run `in production`_. + +Numerous speed achievements are described on `our blog`_. Normalized speed +charts comparing `pypy 1.4 and pypy 1.3`_ as well as `pypy 1.4 and cpython 2.6`_ +are available on benchmark website. For the impatient: yes, we got a lot faster! More highlights =============== -* Virtualenv support: now PyPy is fully compatible with virtualenv_: note that +* PyPy's built-in Just-in-Time compiler is fully transparent and + automatically generated; it now also has very reasonable memory + requirements. The total memory used by a very complex and + long-running process (translating PyPy itself) is within 1.5x to + at most 2x the memory needed by CPython, for a speed-up of 2x. + +* More compact instances. All instances are as compact as if + they had ``__slots__``. This can give programs a big gain in + memory. (In the example of translation above, we already have + carefully placed ``__slots__``, so there is no extra win.) + +* `Virtualenv support`_: now PyPy is fully compatible with virtualenv_: note that to use it, you need a recent version of virtualenv (>= 1.5). * Faster (and JITted) regular expressions - huge boost in speeding up - sre module. + the `re` module. -* Faster (and JITted) calls to functions like map(). +* Other speed improvements, like JITted calls to functions like map(). .. _virtualenv: http://pypi.python.org/pypi/virtualenv +.. _`Virtualenv support`: http://morepypy.blogspot.com/2010/08/using-virtualenv-with-pypy.html +.. _`in production`: http://morepypy.blogspot.com/2010/11/running-large-radio-telescope-software.html +.. _`our blog`: http://morepypy.blogspot.com +.. _`pypy 1.4 and pypy 1.3`: http://speed.pypy.org/comparison/?exe=1%2B41,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=1%2B41&chart=normal+bars +.. _`pypy 1.4 and cpython 2.6`: http://speed.pypy.org/comparison/?exe=2%2B35,1%2B172&ben=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20&env=1&hor=false&bas=2%2B35&chart=normal+bars + +Cheers, + +Carl Friedrich Bolz, Antonio Cuni, Maciej Fijalkowski, +Amaury Forgeot d'Arc, Armin Rigo and the PyPy team Modified: pypy/branch/jit-unroll-loops/pypy/doc/sprint-reports.txt ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/doc/sprint-reports.txt (original) +++ pypy/branch/jit-unroll-loops/pypy/doc/sprint-reports.txt Sat Dec 11 15:10:15 2010 @@ -30,6 +30,17 @@ * `D?sseldorf (October 2006)`_ * `Leysin (January 2007)`_ * `Hildesheim (Feb 2007)`_ (also `EU report writing sprint`_) + * `G?teborg (November 2007)`_ + * `Leysin (January 2008)`_ + * `Berlin (May 2008)`_ + * `Vilnius after EuroPython (July 2008)`_ + * `D?sseldorf (August 2008)`_ + * `Wroclaw (February 2009)`_ + * `Leysin (April 2009)`_ + * `G?teborg (August 2009)`_ + * `D?sseldorf (November 2009)`_ + * `CERN (July 2010)`_ + * `D?sseldorf (October 2010)`_ .. _Hildesheim (Feb 2003): http://codespeak.net/pypy/extradoc/sprintinfo/HildesheimReport.html .. _Gothenburg (May 2003): http://codespeak.net/pypy/extradoc/sprintinfo/gothenburg-2003-sprintreport.txt @@ -55,3 +66,15 @@ .. _Hildesheim (Feb 2007): http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/sprint-report.txt .. _`EU report writing sprint`: http://codespeak.net/pypy/extradoc/sprintinfo/trillke-2007/eu-report-sprint-report.txt .. _`PyCon/Dallas (Feb 2006)`: http://codespeak.net/pypy/extradoc/sprintinfo/pycon06/sprint-report.txt + .. _`G?teborg (November 2007)`: http://morepypy.blogspot.com/2007_11_01_archive.html + .. _`Leysin (January 2008)`: http://morepypy.blogspot.com/2008/01/leysin-winter-sport-sprint-started.html + .. _`Berlin (May 2008)`: http://morepypy.blogspot.com/2008_05_01_archive.html + .. _`Vilnius after EuroPython (July 2008)`: http://morepypy.blogspot.com/2008/07/europython-2008-pypy-talks-and-sprint.html + .. _`D?sseldorf (August 2008)`: http://morepypy.blogspot.com/2008_10_01_archive.html + .. _`Wroclaw (February 2009)`: http://morepypy.blogspot.com/2009/02/wroclaw-2009-sprint-progress-report.html + .. _`Leysin (April 2009)`: http://morepypy.blogspot.com/2009/04/leysin-sprint-report.html + .. _`G?teborg (August 2009)`: http://morepypy.blogspot.com/2009/08/gothenburg-jit-sprint-report.html + .. _`D?sseldorf (November 2009)`: http://morepypy.blogspot.com/2009/11/dusseldorf-sprint-report.html + .. _`CERN (July 2010)`: http://morepypy.blogspot.com/2010/07/cern-sprint-report-wrapping-c-libraries.html + .. _`D?sseldorf (October 2010)`: http://morepypy.blogspot.com/2010/10/dusseldorf-sprint-report-2010.html + Modified: pypy/branch/jit-unroll-loops/pypy/doc/statistic/release_dates.dat ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/doc/statistic/release_dates.dat (original) +++ pypy/branch/jit-unroll-loops/pypy/doc/statistic/release_dates.dat Sat Dec 11 15:10:15 2010 @@ -7,3 +7,7 @@ 2006-06-25,"PyPy 0.9" 2007-02-17,"PyPy 0.99" 2007-03-27,"PyPy 1.0" +2009-04-28,"PyPy 1.1" +2010-03-12,"PyPy 1.2" +2010-06-26,"PyPy 1.3" +2010-11-26,"PyPy 1.4" Modified: pypy/branch/jit-unroll-loops/pypy/doc/statistic/sprint_dates.dat ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/doc/statistic/sprint_dates.dat (original) +++ pypy/branch/jit-unroll-loops/pypy/doc/statistic/sprint_dates.dat Sat Dec 11 15:10:15 2010 @@ -1,26 +1,39 @@ PyPy sprints location, begin, end "Hildesheim",2003-02-17,2003-02-23 -"Gothenburg",2003-05-24,2003-05-31 -"LovainLaNeuve",2003-06-21,2003-06-24 +"G??teborg",2003-05-24,2003-05-31 +"Louvain-la-Neuve",2003-06-21,2003-06-24 "Berlin",2003-09-29,2003-10-04 "Amsterdam",2003-12-14,2003-12-21 -"Europython/Gothenburg",2004-06-01,2004-06-07 +"Europython/G??teborg",2004-06-01,2004-06-07 "Vilnius",2004-11-15,2004-11-23 "Leysin",2005-01-22,2005-01-29 "PyCon/Washington",2005-03-19,2005-03-22 -"Europython/Gothenburg",2005-07-01,2005-07-07 +"Europython/G??teborg",2005-07-01,2005-07-07 "Hildesheim",2005-07-25,2005-07-31 "Heidelberg",2005-08-22,2005-08-29 "Paris",2005-10-10,2005-10-16 -"Gothenburg",2005-12-05,2005-12-11 +"G??teborg",2005-12-05,2005-12-11 "Mallorca",2006-01-23,2006-01-29 "Pycon/Dallas",2006-02-27,2006-03-02 "Louvain-la-Neuve",2006-03-06,2006-03-10 "Japan",2006-04-23,2006-04-29 -"Duesseldorf",2006-06-02,2006-06-09 +"D??sseldorf",2006-06-02,2006-06-09 "Europython/Genf",2006-07-06,2006-07-09 "Limerick",2006-08-21,2006-08-27 -"Duesseldorf",2006-10-30,2006-11-05 +"D??sseldorf",2006-10-30,2006-11-05 "Leysin",2007-01-08,2007-01-14 "Hildesheim",2007-03-01,2007-03-05 +"Hildesheim",2007-03-18,2007-03-23 +"Bern",2007-10-22,2007-10-26 +"G??teborg",2007-11-19,2007-11-25 +"Leysin",2008-01-12,2008-01-19 +"Berlin",2008-05-17,2008-05-22 +"EuroPython/Vilnius",2008-07-10,2008-07-12 +"D??sseldorf",2008-08-05,2008-08-13 +"Wroclaw",2009-02-07,2009-02-14 +"Leysin",2009-04-14,2009-04-21 +"G??teborg",2009-08-18,2009-08-25 +"D??sseldorf",2009-11-06,2009-11-13 +"CERN",2010-07-05,2010-07-09 +"D??sseldorf",2010-10-25,2010-10-31 Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/argument.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/argument.py Sat Dec 11 15:10:15 2010 @@ -402,9 +402,10 @@ space = self.space w_args = space.newtuple(self.arguments_w) w_kwds = space.newdict() - for i in range(len(self.keywords)): - space.setitem(w_kwds, space.wrap(self.keywords[i]), - self.keywords_w[i]) + if self.keywords is not None: + for i in range(len(self.keywords)): + space.setitem(w_kwds, space.wrap(self.keywords[i]), + self.keywords_w[i]) return w_args, w_kwds class ArgumentsForTranslation(Arguments): Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/baseobjspace.py Sat Dec 11 15:10:15 2010 @@ -147,7 +147,7 @@ __already_enqueued_for_destruction = False - def _enqueue_for_destruction(self, space): + def _enqueue_for_destruction(self, space, call_user_del=True): """Put the object in the destructor queue of the space. At a later, safe point in time, UserDelAction will use space.userdel() to call the object's app-level __del__ method. @@ -160,7 +160,8 @@ return self.__already_enqueued_for_destruction = True self.clear_all_weakrefs() - space.user_del_action.register_dying_object(self) + if call_user_del: + space.user_del_action.register_dying_object(self) def _call_builtin_destructor(self): pass # method overridden in typedef.py @@ -861,14 +862,14 @@ def call_args_and_c_profile(self, frame, w_func, args): ec = self.getexecutioncontext() - ec.c_call_trace(frame, w_func) + ec.c_call_trace(frame, w_func, args) try: w_res = self.call_args(w_func, args) except OperationError, e: w_value = e.get_w_value(self) ec.c_exception_trace(frame, w_value) raise - ec.c_return_trace(frame, w_func) + ec.c_return_trace(frame, w_func, args) return w_res def call_method(self, w_obj, methname, *arg_w): @@ -1182,6 +1183,27 @@ self.wrap("expected a 32-bit integer")) return value + def c_filedescriptor_w(self, w_fd): + try: + fd = self.c_int_w(w_fd) + except OperationError, e: + if not e.match(self, self.w_TypeError): + raise + try: + w_fileno = self.getattr(w_fd, self.wrap('fileno')) + except OperationError, e: + if e.match(self, self.w_AttributeError): + raise OperationError(self.w_TypeError, + self.wrap("argument must be an int, " + "or have a fileno() method.")) + raise + w_fd = self.call_function(w_fileno) + fd = self.c_int_w(w_fd) + if fd < 0: + raise operationerrfmt(self.w_ValueError, + "file descriptor cannot be a negative integer (%d)", fd) + return fd + def warn(self, msg, w_warningcls): self.appexec([self.wrap(msg), w_warningcls], """(msg, warningcls): import warnings Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/executioncontext.py Sat Dec 11 15:10:15 2010 @@ -27,7 +27,6 @@ def __init__(self, space): self.space = space self.topframeref = jit.vref_None - self.framestackdepth = 0 # tracing: space.frame_trace_action.fire() must be called to ensure # that tracing occurs whenever self.w_tracefunc or self.is_tracing # is modified. @@ -54,9 +53,6 @@ return frame def enter(self, frame): - if self.framestackdepth > self.space.sys.recursionlimit: - raise self.space.prebuilt_recursion_error - self.framestackdepth += 1 frame.f_backref = self.topframeref self.topframeref = jit.virtual_ref(frame) @@ -66,7 +62,6 @@ self._trace(frame, 'leaveframe', self.space.w_None) finally: self.topframeref = frame.f_backref - self.framestackdepth -= 1 jit.virtual_ref_finish(frame) if self.w_tracefunc is not None and not frame.hide(): @@ -80,7 +75,6 @@ def __init__(self): self.topframe = None - self.framestackdepth = 0 self.w_tracefunc = None self.profilefunc = None self.w_profilefuncarg = None @@ -88,7 +82,6 @@ def enter(self, ec): ec.topframeref = jit.non_virtual_ref(self.topframe) - ec.framestackdepth = self.framestackdepth ec.w_tracefunc = self.w_tracefunc ec.profilefunc = self.profilefunc ec.w_profilefuncarg = self.w_profilefuncarg @@ -97,7 +90,6 @@ def leave(self, ec): self.topframe = ec.gettopframe() - self.framestackdepth = ec.framestackdepth self.w_tracefunc = ec.w_tracefunc self.profilefunc = ec.profilefunc self.w_profilefuncarg = ec.w_profilefuncarg @@ -105,7 +97,6 @@ def clear_framestack(self): self.topframe = None - self.framestackdepth = 0 # the following interface is for pickling and unpickling def getstate(self, space): @@ -121,33 +112,41 @@ self.topframe = space.interp_w(PyFrame, frames_w[-1]) else: self.topframe = None - self.framestackdepth = len(frames_w) def getframestack(self): - index = self.framestackdepth - lst = [None] * index + lst = [] f = self.topframe - while index > 0: - index -= 1 - lst[index] = f + while f is not None: + lst.append(f) f = f.f_backref() - assert f is None + lst.reverse() return lst # coroutine: I think this is all, folks! - def c_call_trace(self, frame, w_func): + def c_call_trace(self, frame, w_func, args=None): "Profile the call of a builtin function" - if self.profilefunc is None: - frame.is_being_profiled = False - else: - self._trace(frame, 'c_call', w_func) + self._c_call_return_trace(frame, w_func, args, 'c_call') - def c_return_trace(self, frame, w_retval): + def c_return_trace(self, frame, w_func, args=None): "Profile the return from a builtin function" + self._c_call_return_trace(frame, w_func, args, 'c_return') + + def _c_call_return_trace(self, frame, w_func, args, event): if self.profilefunc is None: frame.is_being_profiled = False else: - self._trace(frame, 'c_return', w_retval) + # undo the effect of the CALL_METHOD bytecode, which would be + # that even on a built-in method call like '[].append()', + # w_func is actually the unbound function 'append'. + from pypy.interpreter.function import FunctionWithFixedCode + if isinstance(w_func, FunctionWithFixedCode) and args is not None: + w_firstarg = args.firstarg() + if w_firstarg is not None: + from pypy.interpreter.function import descr_function_get + w_func = descr_function_get(self.space, w_func, w_firstarg, + self.space.type(w_firstarg)) + # + self._trace(frame, event, w_func) def c_exception_trace(self, frame, w_exc): "Profile function called upon OperationError." Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/function.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/function.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/function.py Sat Dec 11 15:10:15 2010 @@ -48,7 +48,7 @@ return "<%s %s>" % (self.__class__.__name__, name) def call_args(self, args): - # delegate activation to code + # delegate activation to code return self.getcode().funcrun(self, args) def call_obj_args(self, w_obj, args): @@ -61,17 +61,17 @@ return _get_immutable_code(self) return jit.hint(self.code, promote=True) return self.code - + def funccall(self, *args_w): # speed hack from pypy.interpreter import gateway from pypy.interpreter.pycode import PyCode - + code = self.getcode() # hook for the jit nargs = len(args_w) fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: if nargs == 0: - assert isinstance(code, gateway.BuiltinCode0) + assert isinstance(code, gateway.BuiltinCode0) return code.fastcall_0(self.space, self) elif nargs == 1: assert isinstance(code, gateway.BuiltinCode1) @@ -80,22 +80,22 @@ assert isinstance(code, gateway.BuiltinCode2) return code.fastcall_2(self.space, self, args_w[0], args_w[1]) elif nargs == 3: - assert isinstance(code, gateway.BuiltinCode3) + assert isinstance(code, gateway.BuiltinCode3) return code.fastcall_3(self.space, self, args_w[0], args_w[1], args_w[2]) elif nargs == 4: - assert isinstance(code, gateway.BuiltinCode4) + assert isinstance(code, gateway.BuiltinCode4) return code.fastcall_4(self.space, self, args_w[0], args_w[1], args_w[2], args_w[3]) elif (nargs|PyCode.FLATPYCALL) == fast_natural_arity: - assert isinstance(code, PyCode) + assert isinstance(code, PyCode) if nargs < 5: new_frame = self.space.createframe(code, self.w_func_globals, self.closure) for i in funccallunrolling: if i < nargs: new_frame.fastlocals_w[i] = args_w[i] - return new_frame.run() + return new_frame.run() elif nargs >= 1 and fast_natural_arity == Code.PASSTHROUGHARGS1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) return code.funcrun_obj(self, args_w[0], @@ -106,9 +106,9 @@ def funccall_valuestack(self, nargs, frame): # speed hack from pypy.interpreter import gateway from pypy.interpreter.pycode import PyCode - + code = self.getcode() # hook for the jit - fast_natural_arity = code.fast_natural_arity + fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: if nargs == 0: assert isinstance(code, gateway.BuiltinCode0) @@ -143,7 +143,7 @@ w_obj = frame.peekvalue(nargs-1) args = frame.make_arguments(nargs-1) return code.funcrun_obj(self, w_obj, args) - + args = frame.make_arguments(nargs) return self.call_args(args) @@ -155,8 +155,8 @@ for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) new_frame.fastlocals_w[i] = w_arg - - return new_frame.run() + + return new_frame.run() @jit.unroll_safe def _flat_pycall_defaults(self, code, nargs, frame, defs_to_load): @@ -166,7 +166,7 @@ for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) new_frame.fastlocals_w[i] = w_arg - + defs_w = self.defs_w ndefs = len(defs_w) start = ndefs-defs_to_load @@ -174,7 +174,7 @@ for j in xrange(start, ndefs): new_frame.fastlocals_w[i] = defs_w[j] i += 1 - return new_frame.run() + return new_frame.run() def getdict(self): if self.w_func_dict is None: @@ -188,7 +188,7 @@ # unwrapping is done through unwrap_specs in typedef.py - def descr_function__new__(space, w_subtype, w_code, w_globals, + def descr_function__new__(space, w_subtype, w_code, w_globals, w_name=None, w_argdefs=None, w_closure=None): code = space.interp_w(Code, w_code) if not space.is_true(space.isinstance(w_globals, space.w_dict)): @@ -229,7 +229,7 @@ return self.getrepr(self.space, 'function %s' % (self.name,)) - # delicate + # delicate _all = {'': None} def _freeze_(self): @@ -260,7 +260,7 @@ new_inst = mod.get('builtin_function') return space.newtuple([new_inst, space.newtuple([space.wrap(code.identifier)])]) - + new_inst = mod.get('func_new') w = space.wrap if self.closure is None: @@ -524,7 +524,7 @@ space = self.space other = space.interpclass_w(w_other) if not isinstance(other, Method): - return space.w_False + return space.w_NotImplemented if self.w_instance is None: if other.w_instance is not None: return space.w_False @@ -562,7 +562,7 @@ else: tup = [self.w_function, w_instance, self.w_class] return space.newtuple([new_inst, space.newtuple(tup)]) - + class StaticMethod(Wrappable): """The staticmethod objects.""" _immutable_ = True @@ -612,11 +612,9 @@ self.w_func_dict = func.w_func_dict self.w_module = func.w_module - def descr_builtinfunction__new__(space, w_subtype, w_func): - func = space.interp_w(Function, w_func) - bltin = space.allocate_instance(BuiltinFunction, w_subtype) - BuiltinFunction.__init__(bltin, func) - return space.wrap(bltin) + def descr_builtinfunction__new__(space, w_subtype): + raise OperationError(space.w_TypeError, + space.wrap("cannot create 'builtin_function' instances")) def descr_function_repr(self): return self.space.wrap('' % (self.name,)) Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/gateway.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/gateway.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/gateway.py Sat Dec 11 15:10:15 2010 @@ -1083,7 +1083,7 @@ # these decorators are known to return the same function # object, we may ignore them assert '\n' in source - source = source[source.find('\n') + 1:] + source = source[source.find('\n') + 1:].lstrip() assert source.startswith("def "), "can only transform functions" source = source[4:] p = source.find('(') Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/generator.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/generator.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/generator.py Sat Dec 11 15:10:15 2010 @@ -10,7 +10,7 @@ def __init__(self, frame): self.space = frame.space - self.frame = frame + self.frame = frame # turned into None when frame_finished_execution self.running = False def descr__reduce__(self, space): @@ -19,9 +19,13 @@ mod = space.interp_w(MixedModule, w_mod) new_inst = mod.get('generator_new') w = space.wrap + if self.frame: + w_frame = w(self.frame) + else: + w_frame = space.w_None tup = [ - w(self.frame), + w_frame, w(self.running), ] @@ -41,7 +45,8 @@ if self.running: raise OperationError(space.w_ValueError, space.wrap('generator already executing')) - if self.frame.frame_finished_execution: + frame = self.frame + if frame is None: # xxx a bit ad-hoc, but we don't want to go inside # execute_generator_frame() if the frame is actually finished if operr is None: @@ -49,7 +54,7 @@ raise operr # XXX it's not clear that last_instr should be promoted at all # but as long as it is necessary for call_assembler, let's do it early - last_instr = jit.hint(self.frame.last_instr, promote=True) + last_instr = jit.hint(frame.last_instr, promote=True) if last_instr == -1: if w_arg and not space.is_w(w_arg, space.w_None): msg = "can't send non-None value to a just-started generator" @@ -60,18 +65,19 @@ self.running = True try: try: - w_result = self.frame.execute_generator_frame(w_arg, operr) + w_result = frame.execute_generator_frame(w_arg, operr) except OperationError: # errors finish a frame - self.frame.frame_finished_execution = True + self.frame = None raise # if the frame is now marked as finished, it was RETURNed from - if self.frame.frame_finished_execution: + if frame.frame_finished_execution: + self.frame = None raise OperationError(space.w_StopIteration, space.w_None) else: return w_result # YIELDed finally: - self.frame.f_backref = jit.vref_None + frame.f_backref = jit.vref_None self.running = False def descr_throw(self, w_type, w_val=None, w_tb=None): @@ -115,7 +121,7 @@ raise OperationError(space.w_RuntimeError, space.wrap(msg)) def descr_gi_frame(space, self): - if not self.frame.frame_finished_execution: + if self.frame is not None and not self.frame.frame_finished_execution: return self.frame else: return space.w_None @@ -125,15 +131,17 @@ applevel __del__, which is called at a safe point after the interp-level __del__ enqueued the object for destruction """ - # Only bother raising an exception if the frame is still not - # finished and finally or except blocks are present. - if not self.frame.frame_finished_execution: + self.descr_close() + + def __del__(self): + # Only bother enqueuing self to raise an exception if the frame is + # still not finished and finally or except blocks are present. + must_call_close = False + if self.frame is not None: block = self.frame.lastblock while block is not None: if not isinstance(block, LoopBlock): - self.descr_close() - return + must_call_close = True + break block = block.previous - - def __del__(self): - self._enqueue_for_destruction(self.space) + self._enqueue_for_destruction(self.space, must_call_close) Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/mixedmodule.py Sat Dec 11 15:10:15 2010 @@ -13,6 +13,10 @@ applevel_name = None expose__file__attribute = True + + # The following attribute is None as long as the module has not been + # imported yet, and when it has been, it is mod.__dict__.items() just + # after startup(). w_initialdict = None def __init__(self, space, w_name): @@ -26,8 +30,14 @@ """This is called each time the module is imported or reloaded """ if self.w_initialdict is not None: + # the module was already imported. Refresh its content with + # the saved dict, as done with built-in and extension modules + # on CPython. space.call_method(self.w_dict, 'update', self.w_initialdict) - Module.init(self, space) + else: + Module.init(self, space) + if not self.lazy and self.w_initialdict is None: + self.w_initialdict = space.call_method(self.w_dict, 'items') def get_applevel_name(cls): """ NOT_RPYTHON """ @@ -96,6 +106,7 @@ def _freeze_(self): self.getdict() + self.w_initialdict = None self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/pycode.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/pycode.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/pycode.py Sat Dec 11 15:10:15 2010 @@ -15,7 +15,7 @@ CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_CONTAINSGLOBALS) from pypy.rlib.rarithmetic import intmask -from pypy.rlib.debug import make_sure_not_resized, make_sure_not_modified +from pypy.rlib.debug import make_sure_not_resized from pypy.rlib import jit from pypy.rlib.objectmodel import compute_hash from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT @@ -69,7 +69,7 @@ self.co_stacksize = stacksize self.co_flags = flags self.co_code = code - self.co_consts_w = make_sure_not_modified(consts) + self.co_consts_w = consts self.co_names_w = [space.new_interned_str(aname) for aname in names] self.co_varnames = varnames self.co_freevars = freevars @@ -269,7 +269,7 @@ dis.dis(co) def fget_co_consts(space, self): - return space.newtuple(self.co_consts_w[:]) + return space.newtuple(self.co_consts_w) def fget_co_names(space, self): return space.newtuple(self.co_names_w) @@ -383,7 +383,7 @@ w(self.co_stacksize), w(self.co_flags), w(self.co_code), - space.newtuple(self.co_consts_w[:]), + space.newtuple(self.co_consts_w), space.newtuple(self.co_names_w), space.newtuple([w(v) for v in self.co_varnames]), w(self.co_filename), Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/error.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/error.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/error.py Sat Dec 11 15:10:15 2010 @@ -2,19 +2,22 @@ class SyntaxError(Exception): """Base class for exceptions raised by the parser.""" - def __init__(self, msg, lineno=0, offset=0, text=None, filename=None): + def __init__(self, msg, lineno=0, offset=0, text=None, filename=None, + lastlineno=0): self.msg = msg self.lineno = lineno self.offset = offset self.text = text self.filename = filename + self.lastlineno = lastlineno def wrap_info(self, space): return space.newtuple([space.wrap(self.msg), space.newtuple([space.wrap(self.filename), space.wrap(self.lineno), space.wrap(self.offset), - space.wrap(self.text)])]) + space.wrap(self.text), + space.wrap(self.lastlineno)])]) def __str__(self): return "%s at pos (%d, %d) in %r" % (self.__class__.__name__, @@ -33,8 +36,9 @@ class TokenError(SyntaxError): - def __init__(self, msg, line, lineno, column, tokens): - SyntaxError.__init__(self, msg, lineno, column, line) + def __init__(self, msg, line, lineno, column, tokens, lastlineno=0): + SyntaxError.__init__(self, msg, lineno, column, line, + lastlineno=lastlineno) self.tokens = tokens class TokenIndentationError(IndentationError): Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/pyparser/pytokenizer.py Sat Dec 11 15:10:15 2010 @@ -78,6 +78,7 @@ contline = None indents = [0] last_comment = '' + parenlevstart = (0, 0, "") # make the annotator happy endDFA = automata.DFA([], []) @@ -85,7 +86,7 @@ line = '' pos = 0 lines.append("") - strstart = (0, 0) + strstart = (0, 0, "") for line in lines: lnum = lnum + 1 pos, max = 0, len(line) @@ -93,7 +94,8 @@ if contstr: if not line: raise TokenError("EOF while scanning triple-quoted string", - line, lnum-1, 0, token_list) + strstart[2], strstart[0], strstart[1]+1, + token_list, lnum) endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch @@ -146,6 +148,10 @@ else: # continued statement if not line: + if parenlev > 0: + lnum1, start1, line1 = parenlevstart + raise TokenError("parenthesis is never closed", line1, + lnum1, start1 + 1, token_list, lnum) raise TokenError("EOF in multi-line statement", line, lnum, 0, token_list) continued = 0 @@ -187,7 +193,7 @@ token_list.append(tok) last_comment = '' else: - strstart = (lnum, start) + strstart = (lnum, start, line) contstr = line[start:] contline = line break @@ -195,7 +201,7 @@ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string - strstart = (lnum, start) + strstart = (lnum, start, line) endDFA = (endDFAs[initial] or endDFAs[token[1]] or endDFAs[token[2]]) contstr, needcont = line[start:], 1 @@ -212,6 +218,8 @@ continued = 1 else: if initial in '([{': + if parenlev == 0: + parenlevstart = (lnum, start, line) parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 @@ -230,7 +238,7 @@ start = pos if start', 'exec', 0) @@ -908,3 +919,25 @@ assert e.msg == 'unindent does not match any outer indentation level' else: raise Exception("DID NOT RAISE") + + + def test_repr_vs_str(self): + source1 = "x = (\n" + source2 = "x = (\n\n" + try: + exec source1 + except SyntaxError, err1: + pass + else: + raise Exception("DID NOT RAISE") + try: + exec source2 + except SyntaxError, err2: + pass + else: + raise Exception("DID NOT RAISE") + assert str(err1) != str(err2) + assert repr(err1) != repr(err2) + err3 = eval(repr(err1)) + assert str(err3) == str(err1) + assert repr(err3) == repr(err1) Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_executioncontext.py Sat Dec 11 15:10:15 2010 @@ -7,6 +7,10 @@ class TestExecutionContext: + keywords = {} + + def setup_class(cls): + cls.space = gettestobjspace(**cls.keywords) def test_action(self): @@ -77,29 +81,43 @@ assert l == ['call', 'return', 'call', 'return'] def test_llprofile_c_call(self): + from pypy.interpreter.function import Function, Method l = [] + seen = [] + space = self.space - def profile_func(space, w_arg, frame, event, w_aarg): + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) + if event == 'c_call': + seen.append(w_func) - space = self.space - space.getexecutioncontext().setllprofile(profile_func, space.w_None) - - def check_snippet(snippet): + def check_snippet(snippet, expected_c_call): + del l[:] + del seen[:] + space.getexecutioncontext().setllprofile(profile_func, + space.w_None) space.appexec([], """(): %s return """ % snippet) space.getexecutioncontext().setllprofile(None, None) assert l == ['call', 'return', 'call', 'c_call', 'c_return', 'return'] - - check_snippet('l = []; l.append(42)') - check_snippet('max(1, 2)') - check_snippet('args = (1, 2); max(*args)') - check_snippet('max(1, 2, **{})') - check_snippet('args = (1, 2); max(*args, **{})') - check_snippet('abs(val=0)') + if isinstance(seen[0], Method): + found = 'method %s of %s' % ( + seen[0].w_function.name, + seen[0].w_class.getname(space, '?')) + else: + assert isinstance(seen[0], Function) + found = 'builtin %s' % seen[0].name + assert found == expected_c_call + + check_snippet('l = []; l.append(42)', 'method append of list') + check_snippet('max(1, 2)', 'builtin max') + check_snippet('args = (1, 2); max(*args)', 'builtin max') + check_snippet('max(1, 2, **{})', 'builtin max') + check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') + check_snippet('abs(val=0)', 'builtin abs') def test_llprofile_c_exception(self): l = [] @@ -243,6 +261,13 @@ """) +class TestExecutionContextWithCallLikelyBuiltin(TestExecutionContext): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True} + +class TestExecutionContextWithCallMethod(TestExecutionContext): + keywords = {'objspace.opcodes.CALL_METHOD': True} + + class AppTestDelNotBlocked: def setup_method(self, meth): Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_function.py Sat Dec 11 15:10:15 2010 @@ -6,7 +6,7 @@ from pypy.interpreter.argument import Arguments -class AppTestFunctionIntrospection: +class AppTestFunctionIntrospection: def test_attributes(self): globals()['__name__'] = 'mymodulename' def f(): pass @@ -88,10 +88,10 @@ def f(*args): return 42 raises(TypeError, "dir.func_code = f.func_code") - raises(TypeError, "list.append.im_func.func_code = f.func_code") + raises(TypeError, "list.append.im_func.func_code = f.func_code") -class AppTestFunction: +class AppTestFunction: def test_simple_call(self): def func(arg1, arg2): return arg1, arg2 @@ -116,7 +116,7 @@ assert res[2] == 333 raises(TypeError, func) - raises(TypeError, func, 1, 2, 3, 4) + raises(TypeError, func, 1, 2, 3, 4) def test_simple_varargs(self): def func(arg1, *args): @@ -127,7 +127,7 @@ res = func(23, *(42,)) assert res[0] == 23 - assert res[1] == (42,) + assert res[1] == (42,) def test_simple_kwargs(self): def func(arg1, **kwargs): @@ -205,7 +205,7 @@ func(**{'self': 23}) assert False except TypeError: - pass + pass def test_kwargs_confusing_name(self): def func(self): # 'self' conflicts with the interp-level @@ -287,7 +287,7 @@ # on function types raises(ValueError, type(f).__setstate__, f, (1, 2, 3)) -class AppTestMethod: +class AppTestMethod: def test_simple_call(self): class A(object): def func(self, arg2): @@ -308,7 +308,7 @@ res = a.func(*(42,)) assert res[0] is a - assert res[1] == (42,) + assert res[1] == (42,) def test_obscure_varargs(self): class A(object): @@ -321,14 +321,14 @@ res = a.func(*(42,)) assert res[0] is a - assert res[1] == 42 + assert res[1] == 42 def test_simple_kwargs(self): class A(object): def func(self, **kwargs): return self, kwargs a = A() - + res = a.func(value=42) assert res[0] is a assert res[1] == {'value': 42} @@ -382,19 +382,19 @@ assert hash(C.m) == hash(D.m) assert hash(c.m) == hash(c.m) - def test_method_repr(self): - class A(object): - def f(self): + def test_method_repr(self): + class A(object): + def f(self): pass assert repr(A.f) == "" - assert repr(A().f).startswith(">") + assert repr(A().f).startswith(">") class B: def f(self): pass assert repr(B.f) == "" assert repr(B().f).startswith(">") + assert repr(A().f).endswith(">>") def test_method_call(self): @@ -487,14 +487,33 @@ def f(): pass raises(TypeError, new.instancemethod, f, None) + def test_empty_arg_kwarg_call(self): + def f(): + pass + + raises(TypeError, lambda: f(*0)) + raises(TypeError, lambda: f(**0)) + + def test_method_equal(self): + class A(object): + def m(self): + pass + + class X(object): + def __eq__(self, other): + return True -class TestMethod: + assert A().m == X() + assert X() == A().m + + +class TestMethod: def setup_method(self, method): def c(self, bar): return bar code = PyCode._from_code(self.space, c.func_code) self.fn = Function(self.space, code, self.space.newdict()) - + def test_get(self): space = self.space w_meth = descr_function_get(space, self.fn, space.wrap(5), space.type(space.wrap(5))) @@ -552,7 +571,7 @@ def test_call_function(self): space = self.space - + d = {} for i in range(10): args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")" @@ -574,14 +593,14 @@ code.funcrun = bomb code.funcrun_obj = bomb - args_w = map(space.wrap, range(i)) + args_w = map(space.wrap, range(i)) w_res = space.call_function(fn, *args_w) check = space.is_true(space.eq(w_res, space.wrap(res))) assert check def test_flatcall(self): space = self.space - + def f(a): return a code = PyCode._from_code(self.space, f.func_code) @@ -608,7 +627,7 @@ def test_flatcall_method(self): space = self.space - + def f(self, a): return a code = PyCode._from_code(self.space, f.func_code) @@ -636,7 +655,7 @@ def test_flatcall_default_arg(self): space = self.space - + def f(a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -665,7 +684,7 @@ def test_flatcall_default_arg_method(self): space = self.space - + def f(self, a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -688,7 +707,7 @@ y = A().m(x) b = A().m z = b(x) - return y+10*z + return y+10*z """) assert space.eq_w(w_res, space.wrap(44)) Modified: pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_gateway.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_gateway.py (original) +++ pypy/branch/jit-unroll-loops/pypy/interpreter/test/test_gateway.py Sat Dec 11 15:10:15 2010 @@ -578,6 +578,11 @@ w_res = space.call_args(w_g, args) assert space.eq_w(w_res, space.wrap((-1, 0))) +class AppTestPyTestMark: + @py.test.mark.unlikely_to_exist + def test_anything(self): + pass + class TestPassThroughArguments: Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/detect_cpu.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/detect_cpu.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/detect_cpu.py Sat Dec 11 15:10:15 2010 @@ -31,7 +31,8 @@ 'i86pc': 'x86', # Solaris/Intel 'x86': 'x86', # Apple 'Power Macintosh': 'ppc', - 'x86_64': 'x86', + 'x86_64': 'x86', + 'amd64': 'x86' # freebsd }[mach] except KeyError: return mach Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/llimpl.py Sat Dec 11 15:10:15 2010 @@ -4,6 +4,7 @@ when executing on top of the llinterpreter. """ +import weakref from pypy.objspace.flow.model import Variable, Constant from pypy.annotation import model as annmodel from pypy.jit.metainterp.history import (ConstInt, ConstPtr, @@ -161,6 +162,8 @@ # ____________________________________________________________ class CompiledLoop(object): + has_been_freed = False + def __init__(self): self.inputargs = [] self.operations = [] @@ -285,6 +288,11 @@ del _variables[:] return _to_opaque(CompiledLoop()) +def mark_as_free(loop): + loop = _from_opaque(loop) + assert not loop.has_been_freed + loop.has_been_freed = True + def compile_start_int_var(loop): return compile_start_ref_var(loop, lltype.Signed) @@ -317,7 +325,7 @@ raise ValueError("CALL_ASSEMBLER not supported") loop = _from_opaque(loop) op = loop.operations[-1] - op.descr = descr + op.descr = weakref.ref(descr) def compile_add_var(loop, intvar): loop = _from_opaque(loop) @@ -436,6 +444,7 @@ verbose = True self.opindex = 0 while True: + assert not self.loop.has_been_freed op = self.loop.operations[self.opindex] args = [self.getenv(v) for v in op.args] if not op.is_final(): @@ -447,7 +456,10 @@ _stats.exec_conditional_jumps += 1 if op.jump_target is not None: # a patched guard, pointing to further code - args = [self.getenv(v) for v in op.fail_args if v] + if op.fail_args: + args = [self.getenv(v) for v in op.fail_args if v] + else: + args = [] assert len(op.jump_target.inputargs) == len(args) self.env = dict(zip(op.jump_target.inputargs, args)) self.loop = op.jump_target @@ -578,7 +590,12 @@ def op_debug_merge_point(self, _, value, recdepth): from pypy.jit.metainterp.warmspot import get_stats loc = ConstPtr(value)._get_str() - get_stats().add_merge_point_location(loc) + try: + stats = get_stats() + except AttributeError: + pass + else: + stats.add_merge_point_location(loc) def op_guard_true(self, _, value): if not value: @@ -846,14 +863,22 @@ finally: self._may_force = -1 - def op_call_assembler(self, loop_token, *args): + def op_call_assembler(self, wref_loop_token, *args): + if we_are_translated(): + raise ValueError("CALL_ASSEMBLER not supported") + return self._do_call_assembler(wref_loop_token, *args) + + def _do_call_assembler(self, wref_loop_token, *args): global _last_exception + loop_token = wref_loop_token() + assert loop_token, "CALL_ASSEMBLER to a target that already died" + ctl = loop_token.compiled_loop_token + if hasattr(ctl, 'redirected'): + return self._do_call_assembler(ctl.redirected, *args) assert not self._forced - loop_token = self.cpu._redirected_call_assembler.get(loop_token, - loop_token) self._may_force = self.opindex try: - inpargs = _from_opaque(loop_token._llgraph_compiled_version).inputargs + inpargs = _from_opaque(ctl.compiled_version).inputargs for i, inparg in enumerate(inpargs): TYPE = inparg.concretetype if TYPE is lltype.Signed: @@ -1546,10 +1571,13 @@ do_setfield_gc_int(vable, fielddescr.ofs, 0) def redirect_call_assembler(cpu, oldlooptoken, newlooptoken): - OLD = _from_opaque(oldlooptoken._llgraph_compiled_version).getargtypes() - NEW = _from_opaque(newlooptoken._llgraph_compiled_version).getargtypes() + oldclt = oldlooptoken.compiled_loop_token + newclt = newlooptoken.compiled_loop_token + OLD = _from_opaque(oldclt.compiled_version).getargtypes() + NEW = _from_opaque(newclt.compiled_version).getargtypes() assert OLD == NEW - cpu._redirected_call_assembler[oldlooptoken] = newlooptoken + assert not hasattr(oldclt, 'redirected') + oldclt.redirected = weakref.ref(newlooptoken) # ____________________________________________________________ @@ -1617,6 +1645,7 @@ setannotation(compile_add_fail, annmodel.SomeInteger()) setannotation(compile_add_fail_arg, annmodel.s_None) setannotation(compile_redirect_fail, annmodel.s_None) +setannotation(mark_as_free, annmodel.s_None) setannotation(new_frame, s_Frame) setannotation(frame_clear, annmodel.s_None) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/runner.py Sat Dec 11 15:10:15 2010 @@ -102,7 +102,6 @@ llimpl._llinterp = LLInterpreter(self.rtyper) self._future_values = [] self._descrs = {} - self._redirected_call_assembler = {} def _freeze_(self): assert self.translate_support_code @@ -118,22 +117,34 @@ self._descrs[key] = descr return descr - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): c = llimpl.compile_start() + clt = original_loop_token.compiled_loop_token + clt.loop_and_bridges.append(c) + clt.compiling_a_bridge() self._compile_loop_or_bridge(c, inputargs, operations) old, oldindex = faildescr._compiled_fail llimpl.compile_redirect_fail(old, oldindex, c) - def compile_loop(self, inputargs, operations, loopdescr, log=True): + def compile_loop(self, inputargs, operations, looptoken, log=True): """In a real assembler backend, this should assemble the given list of operations. Here we just generate a similar CompiledLoop instance. The code here is RPython, whereas the code in llimpl is not. """ c = llimpl.compile_start() - loopdescr._llgraph_compiled_version = c + clt = model.CompiledLoopToken(self, looptoken.number) + clt.loop_and_bridges = [c] + clt.compiled_version = c + looptoken.compiled_loop_token = clt self._compile_loop_or_bridge(c, inputargs, operations) + def free_loop_and_bridges(self, compiled_loop_token): + for c in compiled_loop_token.loop_and_bridges: + llimpl.mark_as_free(c) + model.AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) + def _compile_loop_or_bridge(self, c, inputargs, operations): var2index = {} for box in inputargs: @@ -215,7 +226,7 @@ if op.getopnum() == rop.JUMP: targettoken = op.getdescr() assert isinstance(targettoken, history.LoopToken) - compiled_version = targettoken._llgraph_compiled_version + compiled_version = targettoken.compiled_loop_token.compiled_version llimpl.compile_add_jump_target(c, compiled_version) elif op.getopnum() == rop.FINISH: faildescr = op.getdescr() @@ -225,7 +236,7 @@ assert False, "unknown operation" def _execute_token(self, loop_token): - compiled_version = loop_token._llgraph_compiled_version + compiled_version = loop_token.compiled_loop_token.compiled_version frame = llimpl.new_frame(self.is_oo, self) # setup the frame llimpl.frame_clear(frame, compiled_version) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/gc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/gc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/gc.py Sat Dec 11 15:10:15 2010 @@ -15,6 +15,7 @@ from pypy.jit.backend.llsupport.descr import GcCache, get_field_descr from pypy.jit.backend.llsupport.descr import GcPtrFieldDescr from pypy.jit.backend.llsupport.descr import get_call_descr +from pypy.rpython.memory.gctransform import asmgcroot # ____________________________________________________________ @@ -35,6 +36,8 @@ return False def has_write_barrier_class(self): return None + def freeing_block(self, start, stop): + pass # ____________________________________________________________ @@ -218,50 +221,120 @@ LOC_EBP_PLUS = 2 LOC_EBP_MINUS = 3 - GCMAP_ARRAY = rffi.CArray(llmemory.Address) - CALLSHAPE_ARRAY = rffi.CArray(rffi.UCHAR) + GCMAP_ARRAY = rffi.CArray(lltype.Signed) + CALLSHAPE_ARRAY_PTR = rffi.CArrayPtr(rffi.UCHAR) def __init__(self): + # '_gcmap' is an array of length '_gcmap_maxlength' of addresses. + # '_gcmap_curlength' tells how full the array really is. + # The addresses are actually grouped in pairs: + # (addr-after-the-CALL-in-assembler, addr-of-the-call-shape). + # '_gcmap_deadentries' counts pairs marked dead (2nd item is NULL). + # '_gcmap_sorted' is True only if we know the array is sorted. self._gcmap = lltype.nullptr(self.GCMAP_ARRAY) self._gcmap_curlength = 0 self._gcmap_maxlength = 0 + self._gcmap_deadentries = 0 + self._gcmap_sorted = True def initialize(self): # hack hack hack. Remove these lines and see MissingRTypeAttribute # when the rtyper tries to annotate these methods only when GC-ing... self.gcmapstart() self.gcmapend() + self.gcmarksorted() def gcmapstart(self): - return llmemory.cast_ptr_to_adr(self._gcmap) + return rffi.cast(llmemory.Address, self._gcmap) def gcmapend(self): addr = self.gcmapstart() if self._gcmap_curlength: - addr += llmemory.sizeof(llmemory.Address)*self._gcmap_curlength + addr += rffi.sizeof(lltype.Signed) * self._gcmap_curlength + if not we_are_translated() and type(addr) is long: + from pypy.rpython.lltypesystem import ll2ctypes + addr = ll2ctypes._lladdress(addr) # XXX workaround return addr + def gcmarksorted(self): + # Called by the GC when it is about to sort [gcmapstart():gcmapend()]. + # Returns the previous sortedness flag -- i.e. returns True if it + # is already sorted, False if sorting is needed. + sorted = self._gcmap_sorted + self._gcmap_sorted = True + return sorted + def put(self, retaddr, callshapeaddr): """'retaddr' is the address just after the CALL. - 'callshapeaddr' is the address returned by encode_callshape().""" + 'callshapeaddr' is the address of the raw 'shape' marker. + Both addresses are actually integers here.""" index = self._gcmap_curlength if index + 2 > self._gcmap_maxlength: - self._enlarge_gcmap() + index = self._enlarge_gcmap() self._gcmap[index] = retaddr self._gcmap[index+1] = callshapeaddr self._gcmap_curlength = index + 2 + self._gcmap_sorted = False + @rgc.no_collect def _enlarge_gcmap(self): - newlength = 250 + self._gcmap_maxlength * 2 - newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', - track_allocation=False) oldgcmap = self._gcmap - for i in range(self._gcmap_curlength): - newgcmap[i] = oldgcmap[i] - self._gcmap = newgcmap - self._gcmap_maxlength = newlength - if oldgcmap: - lltype.free(oldgcmap, flavor='raw', track_allocation=False) + if self._gcmap_deadentries * 3 * 2 > self._gcmap_maxlength: + # More than 1/3rd of the entries are dead. Don't actually + # enlarge the gcmap table, but just clean up the dead entries. + newgcmap = oldgcmap + else: + # Normal path: enlarge the array. + newlength = 250 + (self._gcmap_maxlength // 3) * 4 + newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', + track_allocation=False) + self._gcmap_maxlength = newlength + # + j = 0 + i = 0 + end = self._gcmap_curlength + while i < end: + if oldgcmap[i + 1]: + newgcmap[j] = oldgcmap[i] + newgcmap[j + 1] = oldgcmap[i + 1] + j += 2 + i += 2 + self._gcmap_curlength = j + self._gcmap_deadentries = 0 + if oldgcmap != newgcmap: + self._gcmap = newgcmap + if oldgcmap: + lltype.free(oldgcmap, flavor='raw', track_allocation=False) + return j + + @rgc.no_collect + def freeing_block(self, start, stop): + # if [start:stop] is a raw block of assembler, then look up the + # corresponding gcroot markers, and mark them as freed now in + # self._gcmap by setting the 2nd address of every entry to NULL. + gcmapstart = self.gcmapstart() + gcmapend = self.gcmapend() + if gcmapstart == gcmapend: + return + if not self.gcmarksorted(): + asmgcroot.sort_gcmap(gcmapstart, gcmapend) + # A note about gcmarksorted(): the deletion we do here keeps the + # array sorted. This avoids needing too many sort_gcmap()s. + # Indeed, freeing_block() is typically called many times in a row, + # so it will call sort_gcmap() at most the first time. + startaddr = rffi.cast(llmemory.Address, start) + stopaddr = rffi.cast(llmemory.Address, stop) + item = asmgcroot.binary_search(gcmapstart, gcmapend, startaddr) + # 'item' points to one of the entries. Because the whole array + # is sorted, we know that it points either to the first entry we + # want to kill, or to the previous entry. + if item.address[0] < startaddr: + item += asmgcroot.arrayitemsize # go forward one entry + assert item == gcmapend or item.address[0] >= startaddr + while item != gcmapend and item.address[0] < stopaddr: + item.address[1] = llmemory.NULL + self._gcmap_deadentries += 1 + item += asmgcroot.arrayitemsize def get_basic_shape(self, is_64_bit=False): # XXX: Should this code even really know about stack frame layout of @@ -304,17 +377,15 @@ assert reg_index > 0 shape.append(chr(self.LOC_REG | (reg_index << 2))) - def compress_callshape(self, shape): + def compress_callshape(self, shape, datablockwrapper): # Similar to compress_callshape() in trackgcroot.py. - # XXX so far, we always allocate a new small array (we could regroup - # them inside bigger arrays) and we never try to share them. + # Returns an address to raw memory (as an integer). length = len(shape) - compressed = lltype.malloc(self.CALLSHAPE_ARRAY, length, - flavor='raw', - track_allocation=False) # memory leak + rawaddr = datablockwrapper.malloc_aligned(length, 1) + p = rffi.cast(self.CALLSHAPE_ARRAY_PTR, rawaddr) for i in range(length): - compressed[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) - return llmemory.cast_ptr_to_adr(compressed) + p[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) + return rawaddr class WriteBarrierDescr(AbstractDescr): @@ -379,6 +450,7 @@ 'layoutbuilder': self.layoutbuilder, 'gcmapstart': lambda: gcrootmap.gcmapstart(), 'gcmapend': lambda: gcrootmap.gcmapend(), + 'gcmarksorted': lambda: gcrootmap.gcmarksorted(), } self.GCClass = self.layoutbuilder.GCClass self.moving_gc = self.GCClass.moving_gc @@ -641,6 +713,9 @@ def has_write_barrier_class(self): return WriteBarrierDescr + def freeing_block(self, start, stop): + self.gcrootmap.freeing_block(start, stop) + # ____________________________________________________________ def get_ll_description(gcdescr, translator=None, rtyper=None): Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/llmodel.py Sat Dec 11 15:10:15 2010 @@ -17,7 +17,7 @@ from pypy.jit.backend.llsupport.descr import get_call_descr from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr -from pypy.jit.backend.llsupport.ffisupport import get_call_descr_dynamic +from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -52,6 +52,7 @@ else: self._setup_exception_handling_untranslated() self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) + self.asmmemmgr = AsmMemoryManager() self.setup() if translate_support_code: self._setup_on_leave_jitted_translated() @@ -177,6 +178,15 @@ self.saved_exc_value = lltype.nullptr(llmemory.GCREF.TO) return exc + def free_loop_and_bridges(self, compiled_loop_token): + AbstractCPU.free_loop_and_bridges(self, compiled_loop_token) + blocks = compiled_loop_token.asmmemmgr_blocks + if blocks is not None: + compiled_loop_token.asmmemmgr_blocks = None + for rawstart, rawstop in blocks: + self.gc_ll_descr.freeing_block(rawstart, rawstop) + self.asmmemmgr.free(rawstart, rawstop) + # ------------------- helpers and descriptions -------------------- @staticmethod @@ -236,7 +246,9 @@ return get_call_descr(self.gc_ll_descr, ARGS, RESULT, extrainfo) def calldescrof_dynamic(self, ffi_args, ffi_result, extrainfo=None): - return get_call_descr_dynamic(ffi_args, ffi_result, extrainfo) + from pypy.jit.backend.llsupport import ffisupport + return ffisupport.get_call_descr_dynamic(ffi_args, ffi_result, + extrainfo) def get_overflow_error(self): ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/test/test_gc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/test/test_gc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/llsupport/test/test_gc.py Sat Dec 11 15:10:15 2010 @@ -9,6 +9,7 @@ from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE from pypy.jit.metainterp.test.test_optimizeopt import equaloplists +from pypy.rpython.memory.gctransform import asmgcroot def test_boehm(): gc_ll_descr = GcLLDescr_boehm(None, None, None) @@ -62,58 +63,169 @@ for i in range(len(allocs)): assert addrs[i].address[0] == llmemory.cast_ptr_to_adr(allocs[i]) -def test_GcRootMap_asmgcc(): - def frame_pos(n): - return -4*(4+n) - gcrootmap = GcRootMap_asmgcc() - num1 = frame_pos(-5) - num1a = num1|2 - num2 = frame_pos(55) - num2a = ((-num2|3) >> 7) | 128 - num2b = (-num2|3) & 127 - shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, num1) - gcrootmap.add_ebp_offset(shape, num2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) - gcrootmap.add_callee_save_reg(shape, 1) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4]) - gcrootmap.add_callee_save_reg(shape, 2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8]) - gcrootmap.add_callee_save_reg(shape, 3) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12]) - gcrootmap.add_callee_save_reg(shape, 4) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12, 16]) - # - shapeaddr = gcrootmap.compress_callshape(shape) - PCALLSHAPE = lltype.Ptr(GcRootMap_asmgcc.CALLSHAPE_ARRAY) - p = llmemory.cast_adr_to_ptr(shapeaddr, PCALLSHAPE) - for i, expected in enumerate([16, 12, 8, 4, - num2a, num2b, num1a, 0, 2, 15, 11, 7, 6]): - assert p[i] == expected - # - retaddr = rffi.cast(llmemory.Address, 1234567890) - gcrootmap.put(retaddr, shapeaddr) - assert gcrootmap._gcmap[0] == retaddr - assert gcrootmap._gcmap[1] == shapeaddr - assert gcrootmap.gcmapstart().address[0] == retaddr - # - # the same as before, but enough times to trigger a few resizes - expected_shapeaddr = {} - for i in range(1, 700): +class TestGcRootMapAsmGcc: + + def test_make_shapes(self): + def frame_pos(n): + return -4*(4+n) + gcrootmap = GcRootMap_asmgcc() + num1 = frame_pos(-5) + num1a = num1|2 + num2 = frame_pos(55) + num2a = ((-num2|3) >> 7) | 128 + num2b = (-num2|3) & 127 shape = gcrootmap.get_basic_shape() - gcrootmap.add_ebp_offset(shape, frame_pos(i)) - shapeaddr = gcrootmap.compress_callshape(shape) - expected_shapeaddr[i] = shapeaddr - retaddr = rffi.cast(llmemory.Address, 123456789 + i) + gcrootmap.add_ebp_offset(shape, num1) + gcrootmap.add_ebp_offset(shape, num2) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) + gcrootmap.add_callee_save_reg(shape, 1) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4]) + gcrootmap.add_callee_save_reg(shape, 2) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8]) + gcrootmap.add_callee_save_reg(shape, 3) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8, 12]) + gcrootmap.add_callee_save_reg(shape, 4) + assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, + 4, 8, 12, 16]) + + def test_compress_callshape(self): + class FakeDataBlockWrapper: + def malloc_aligned(self, size, alignment): + assert alignment == 1 # here + assert size == 4 + return rffi.cast(lltype.Signed, p) + datablockwrapper = FakeDataBlockWrapper() + p = lltype.malloc(rffi.CArray(lltype.Char), 4, immortal=True) + gcrootmap = GcRootMap_asmgcc() + shape = ['a', 'b', 'c', 'd'] + gcrootmap.compress_callshape(shape, datablockwrapper) + assert p[0] == 'd' + assert p[1] == 'c' + assert p[2] == 'b' + assert p[3] == 'a' + + def test_put_basic(self): + gcrootmap = GcRootMap_asmgcc() + retaddr = 1234567890 + shapeaddr = 51627384 gcrootmap.put(retaddr, shapeaddr) - for i in range(1, 700): - expected_retaddr = rffi.cast(llmemory.Address, 123456789 + i) - assert gcrootmap._gcmap[i*2+0] == expected_retaddr - assert gcrootmap._gcmap[i*2+1] == expected_shapeaddr[i] + assert gcrootmap._gcmap[0] == retaddr + assert gcrootmap._gcmap[1] == shapeaddr + p = rffi.cast(rffi.LONGP, gcrootmap.gcmapstart()) + assert p[0] == retaddr + assert (gcrootmap.gcmapend() == + gcrootmap.gcmapstart() + rffi.sizeof(lltype.Signed) * 2) + + def test_put_resize(self): + # the same as before, but enough times to trigger a few resizes + gcrootmap = GcRootMap_asmgcc() + for i in range(700): + shapeaddr = i * 100 + 1 + retaddr = 123456789 + i + gcrootmap.put(retaddr, shapeaddr) + for i in range(700): + assert gcrootmap._gcmap[i*2+0] == 123456789 + i + assert gcrootmap._gcmap[i*2+1] == i * 100 + 1 + + def test_remove_nulls(self): + expected = [] + def check(): + assert gcrootmap._gcmap_curlength == len(expected) * 2 + for i, (a, b) in enumerate(expected): + assert gcrootmap._gcmap[i*2] == a + assert gcrootmap._gcmap[i*2+1] == b + # + gcrootmap = GcRootMap_asmgcc() + for i in range(700): + shapeaddr = i * 100 # 0 if i == 0 + retaddr = 123456789 + i + gcrootmap.put(retaddr, shapeaddr) + if shapeaddr != 0: + expected.append((retaddr, shapeaddr)) + # at the first resize, the 0 should be removed + check() + for repeat in range(10): + # now clear up half the entries + assert len(expected) == 699 + for i in range(0, len(expected), 2): + gcrootmap._gcmap[i*2+1] = 0 + gcrootmap._gcmap_deadentries += 1 + expected = expected[1::2] + assert gcrootmap._gcmap_deadentries*6 > gcrootmap._gcmap_maxlength + # check that we can again insert 350 entries without a resize + oldgcmap = gcrootmap._gcmap + for i in range(0, 699, 2): + gcrootmap.put(515151 + i + repeat, 626262 + i) + expected.append((515151 + i + repeat, 626262 + i)) + assert gcrootmap._gcmap == oldgcmap + check() + + def test_freeing_block(self): + from pypy.jit.backend.llsupport import gc + class Asmgcroot: + arrayitemsize = 2 * llmemory.sizeof(llmemory.Address) + sort_count = 0 + def sort_gcmap(self, gcmapstart, gcmapend): + self.sort_count += 1 + def binary_search(self, gcmapstart, gcmapend, startaddr): + i = 0 + while (i < gcrootmap._gcmap_curlength//2 and + gcrootmap._gcmap[i*2] < startaddr): + i += 1 + if i > 0: + i -= 1 + assert 0 <= i < gcrootmap._gcmap_curlength//2 + p = rffi.cast(rffi.CArrayPtr(llmemory.Address), gcmapstart) + p = rffi.ptradd(p, 2*i) + return llmemory.cast_ptr_to_adr(p) + saved = gc.asmgcroot + try: + gc.asmgcroot = Asmgcroot() + # + gcrootmap = GcRootMap_asmgcc() + gcrootmap._gcmap = lltype.malloc(gcrootmap.GCMAP_ARRAY, + 1400, flavor='raw', + immortal=True) + for i in range(700): + gcrootmap._gcmap[i*2] = 1200000 + i + gcrootmap._gcmap[i*2+1] = i * 100 + 1 + assert gcrootmap._gcmap_deadentries == 0 + assert gc.asmgcroot.sort_count == 0 + gcrootmap._gcmap_maxlength = 1400 + gcrootmap._gcmap_curlength = 1400 + gcrootmap._gcmap_sorted = False + # + gcrootmap.freeing_block(1200000 - 100, 1200000) + assert gcrootmap._gcmap_deadentries == 0 + assert gc.asmgcroot.sort_count == 1 + # + gcrootmap.freeing_block(1200000 + 100, 1200000 + 200) + assert gcrootmap._gcmap_deadentries == 100 + assert gc.asmgcroot.sort_count == 1 + for i in range(700): + if 100 <= i < 200: + expected = 0 + else: + expected = i * 100 + 1 + assert gcrootmap._gcmap[i*2] == 1200000 + i + assert gcrootmap._gcmap[i*2+1] == expected + # + gcrootmap.freeing_block(1200000 + 650, 1200000 + 750) + assert gcrootmap._gcmap_deadentries == 150 + assert gc.asmgcroot.sort_count == 1 + for i in range(700): + if 100 <= i < 200 or 650 <= i: + expected = 0 + else: + expected = i * 100 + 1 + assert gcrootmap._gcmap[i*2] == 1200000 + i + assert gcrootmap._gcmap[i*2+1] == expected + # + finally: + gc.asmgcroot = saved class FakeLLOp(object): Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/model.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/model.py Sat Dec 11 15:10:15 2010 @@ -1,3 +1,4 @@ +from pypy.rlib.debug import debug_start, debug_print, debug_stop from pypy.jit.metainterp import history, compile @@ -7,17 +8,27 @@ done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + total_compiled_loops = 0 + total_compiled_bridges = 0 + total_freed_loops = 0 + total_freed_bridges = 0 def __init__(self): self.fail_descr_list = [] + self.fail_descr_free_list = [] def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) n = descr.index if n < 0: lst = self.fail_descr_list - n = len(lst) - lst.append(descr) + if len(self.fail_descr_free_list) > 0: + n = self.fail_descr_free_list.pop() + assert lst[n] is None + lst[n] = descr + else: + n = len(lst) + lst.append(descr) descr.index = n return n @@ -35,12 +46,14 @@ def compile_loop(self, inputargs, operations, looptoken, log=True): """Assemble the given loop. - Extra attributes should be put in the LoopToken to - point to the compiled loop in assembler. + Should create and attach a fresh CompiledLoopToken to + looptoken.compiled_loop_token and stick extra attributes + on it to point to the compiled loop in assembler. """ raise NotImplementedError - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): """Assemble the bridge. The FailDescr is the descr of the original guard that failed. """ @@ -113,6 +126,28 @@ oldlooptoken so that from now own they will call newlooptoken.""" raise NotImplementedError + def free_loop_and_bridges(self, compiled_loop_token): + """This method is called to free resources (machine code, + references to resume guards, etc.) allocated by the compilation + of a loop and all bridges attached to it. After this call, the + frontend cannot use this compiled loop any more; in fact, it + guarantees that at the point of the call to free_code_group(), + none of the corresponding assembler is currently running. + """ + # The base class provides a limited implementation: freeing the + # resume descrs. This is already quite helpful, because the + # resume descrs are the largest consumers of memory (about 3x + # more than the assembler, in the case of the x86 backend). + lst = self.fail_descr_list + # We expect 'compiled_loop_token' to be itself garbage-collected soon, + # but better safe than sorry: be ready to handle several calls to + # free_loop_and_bridges() for the same compiled_loop_token. + faildescr_indices = compiled_loop_token.faildescr_indices + compiled_loop_token.faildescr_indices = [] + for n in faildescr_indices: + lst[n] = None + self.fail_descr_free_list.extend(faildescr_indices) + @staticmethod def sizeof(S): raise NotImplementedError @@ -237,3 +272,40 @@ def force(self, force_token): raise NotImplementedError + + +class CompiledLoopToken(object): + asmmemmgr_blocks = None + asmmemmgr_gcroots = 0 + + def __init__(self, cpu, number): + cpu.total_compiled_loops += 1 + self.cpu = cpu + self.number = number + self.bridges_count = 0 + # This growing list gives the 'descr_number' of all fail descrs + # that belong to this loop or to a bridge attached to it. + # Filled by the frontend calling record_faildescr_index(). + self.faildescr_indices = [] + debug_start("jit-mem-looptoken-alloc") + debug_print("allocating Loop #", self.number) + debug_stop("jit-mem-looptoken-alloc") + + def record_faildescr_index(self, n): + self.faildescr_indices.append(n) + + def compiling_a_bridge(self): + self.cpu.total_compiled_bridges += 1 + self.bridges_count += 1 + debug_start("jit-mem-looptoken-alloc") + debug_print("allocating Bridge #", self.bridges_count, "of Loop #", self.number) + debug_stop("jit-mem-looptoken-alloc") + + def __del__(self): + debug_start("jit-mem-looptoken-free") + debug_print("freeing Loop #", self.number, 'with', + self.bridges_count, 'attached bridges') + self.cpu.free_loop_and_bridges(self) + self.cpu.total_freed_loops += 1 + self.cpu.total_freed_bridges += self.bridges_count + debug_stop("jit-mem-looptoken-free") Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/test/runner_test.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/test/runner_test.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/test/runner_test.py Sat Dec 11 15:10:15 2010 @@ -174,6 +174,8 @@ assert not wr_i1() and not wr_guard() def test_compile_bridge(self): + self.cpu.total_compiled_loops = 0 + self.cpu.total_compiled_bridges = 0 i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() @@ -199,7 +201,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) @@ -207,6 +209,10 @@ res = self.cpu.get_latest_value_int(0) assert res == 20 + assert self.cpu.total_compiled_loops == 1 + assert self.cpu.total_compiled_bridges == 1 + return looptoken + def test_compile_bridge_with_holes(self): i0 = BoxInt() i1 = BoxInt() @@ -233,7 +239,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) self.cpu.set_future_value_int(0, 2) fail = self.cpu.execute_token(looptoken) @@ -1050,7 +1056,7 @@ ResOperation(rop.JUMP, [f3] + fboxes2[1:], None, descr=looptoken), ] - self.cpu.compile_bridge(faildescr1, fboxes2, bridge) + self.cpu.compile_bridge(faildescr1, fboxes2, bridge, looptoken) for i in range(len(fboxes)): self.cpu.set_future_value_float(i, 13.5 + 6.73 * i) @@ -1197,6 +1203,13 @@ yield nan_and_infinity, rop.FLOAT_GT, operator.gt, all_cases_binary yield nan_and_infinity, rop.FLOAT_GE, operator.ge, all_cases_binary + def test_noops(self): + c_box = self.alloc_string("hi there").constbox() + c_nest = ConstInt(0) + self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest], 'void') + self.execute_operation(rop.JIT_DEBUG, [c_box, c_nest, c_nest, + c_nest, c_nest], 'void') + class LLtypeBackendTest(BaseBackendTest): @@ -2224,6 +2237,20 @@ assert res.value == expected, ( "%r: got %r, expected %r" % (RESTYPE, res.value, expected)) + def test_free_loop_and_bridges(self): + from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("not a subclass of llmodel.AbstractLLCPU") + if hasattr(self.cpu, 'setup_once'): + self.cpu.setup_once() + mem0 = self.cpu.asmmemmgr.total_mallocs + looptoken = self.test_compile_bridge() + mem1 = self.cpu.asmmemmgr.total_mallocs + self.cpu.free_loop_and_bridges(looptoken.compiled_loop_token) + mem2 = self.cpu.asmmemmgr.total_mallocs + assert mem2 < mem1 + assert mem2 == mem0 + class OOtypeBackendTest(BaseBackendTest): Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/test/test_random.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/test/test_random.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/test/test_random.py Sat Dec 11 15:10:15 2010 @@ -524,7 +524,7 @@ self.prebuilt_ptr_consts = [] self.r = r self.build_random_loop(cpu, builder_factory, r, startvars) - + def build_random_loop(self, cpu, builder_factory, r, startvars): loop = TreeLoop('test_random_function') @@ -685,11 +685,12 @@ subloop.operations[-1] = jump_op self.guard_op = rl.guard_op self.prebuilt_ptr_consts += rl.prebuilt_ptr_consts + self.loop.token.record_jump_to(rl.loop.token) self.dont_generate_more = True if r.random() < .05: return False self.builder.cpu.compile_bridge(fail_descr, fail_args, - subloop.operations) + subloop.operations, self.loop.token) return True def check_random_function(cpu, BuilderClass, r, num=None, max=None): Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/assembler.py Sat Dec 11 15:10:15 2010 @@ -1,12 +1,13 @@ import sys, os from pypy.jit.backend.llsupport import symbolic +from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT, LoopToken) from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper -from pypy.tool.uid import fixid +from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, X86RegisterManager, X86XMMRegisterManager, get_ebp_ofs, _get_scale) @@ -30,10 +31,11 @@ from pypy.jit.backend.x86 import rx86, regloc, codebuf from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86.support import values_array -from pypy.rlib.debug import debug_print +from pypy.jit.backend.x86 import support +from pypy.rlib.debug import (debug_print, debug_start, debug_stop, + have_debug_prints) from pypy.rlib import rgc from pypy.jit.backend.x86.jump import remap_frame_layout -from pypy.rlib.streamio import open_file_as_stream from pypy.jit.metainterp.history import ConstInt, BoxInt # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, @@ -43,123 +45,17 @@ def align_stack_words(words): return (words + CALL_ALIGN - 1) & ~(CALL_ALIGN-1) -class MachineCodeBlockWrapper(object): - MC_DEFAULT_SIZE = 1024*1024 - - def __init__(self, assembler, bigsize, profile_agent=None): - self.assembler = assembler - self.old_mcs = [] # keepalive - self.bigsize = bigsize - self._mc = self._instantiate_mc() - self.function_name = None - self.profile_agent = profile_agent - self.reset_reserved_bytes() - - def _instantiate_mc(self): # hook for testing - return codebuf.MachineCodeBlock(self.bigsize) - - def ensure_bytes_available(self, num_bytes): - if self.bytes_free() <= (self._reserved_bytes + num_bytes): - self.make_new_mc() - - def reserve_bytes(self, num_bytes): - self.ensure_bytes_available(num_bytes) - self._reserved_bytes += num_bytes - - def reset_reserved_bytes(self): - # XXX er.... pretty random number, just to be sure - # not to write half-instruction - self._reserved_bytes = 64 - - def get_relative_pos(self): - return self._mc.get_relative_pos() - - def overwrite(self, pos, listofchars): - return self._mc.overwrite(pos, listofchars) - - def bytes_free(self): - return self._mc._size - self._mc.get_relative_pos() - - def start_function(self, name): - self.function_name = name - self.start_pos = self._mc.get_relative_pos() - - def end_function(self, done=True): - assert self.function_name is not None - size = self._mc.get_relative_pos() - self.start_pos - address = self.tell() - size - if self.profile_agent is not None: - self.profile_agent.native_code_written(self.function_name, - address, size) - if done: - self.function_name = None - - def make_new_mc(self): - new_mc = self._instantiate_mc() - debug_print('[new machine code block at', new_mc.tell(), ']') - - if IS_X86_64: - # The scratch register is sometimes used as a temporary - # register, but the JMP below might clobber it. Rather than risk - # subtle bugs, we preserve the scratch register across the jump. - self._mc.PUSH_r(X86_64_SCRATCH_REG.value) - - self._mc.JMP(imm(new_mc.tell())) - - if IS_X86_64: - # Restore scratch reg - new_mc.POP_r(X86_64_SCRATCH_REG.value) - - if self.function_name is not None: - self.end_function(done=False) - self.start_pos = new_mc.get_relative_pos() - - self.assembler.write_pending_failure_recoveries() - - self._mc.done() - self.old_mcs.append(self._mc) - self._mc = new_mc - make_new_mc._dont_inline_ = True - - def tell(self): - return self._mc.tell() - - def done(self): - self._mc.done() - -def _new_method(name): - def method(self, *args): - if self.bytes_free() < self._reserved_bytes: - self.make_new_mc() - getattr(self._mc, name)(*args) - method.func_name = name - return method - -for _name in rx86.all_instructions + regloc.all_extra_instructions: - setattr(MachineCodeBlockWrapper, _name, _new_method(_name)) - -for name in dir(codebuf.MachineCodeBlock): - if name.upper() == name or name == "writechr": - setattr(MachineCodeBlockWrapper, name, _new_method(name)) class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, exc, desc_bytes): + def __init__(self, faildescr, failargs, fail_locs, exc): self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs self.exc = exc - self.desc_bytes = desc_bytes - - def recovery_stub_size(self): - # XXX: 32 is pulled out of the air - return 32 + len(self.desc_bytes) DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed)) class Assembler386(object): - mc = None - mc_size = MachineCodeBlockWrapper.MC_DEFAULT_SIZE - _float_constants = None _regalloc = None _output_loop_log = None @@ -177,16 +73,18 @@ self.fail_boxes_float = values_array(lltype.Float, failargs_limit) self.fail_ebp = 0 self.loop_run_counters = [] - # if we have 10000 loops, we have some other problems I guess self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 self.malloc_fixedsize_slowpath1 = 0 self.malloc_fixedsize_slowpath2 = 0 - self.pending_guard_tokens = None self.memcpy_addr = 0 self.setup_failure_recovery() self._debug = False self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.fail_boxes_count = 0 + self._current_depths_cache = (0, 0) + self.datablockwrapper = None + self.teardown() def leave_jitted_hook(self): ptrs = self.fail_boxes_ptr.ar @@ -196,67 +94,65 @@ def set_debug(self, v): self._debug = v - def setup(self): - if self.mc is None: - # the address of the function called by 'new' - gc_ll_descr = self.cpu.gc_ll_descr - gc_ll_descr.initialize() - ll_new = gc_ll_descr.get_funcptr_for_new() - self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) - if gc_ll_descr.get_funcptr_for_newarray is not None: - ll_new_array = gc_ll_descr.get_funcptr_for_newarray() - self.malloc_array_func_addr = rffi.cast(lltype.Signed, - ll_new_array) - if gc_ll_descr.get_funcptr_for_newstr is not None: - ll_new_str = gc_ll_descr.get_funcptr_for_newstr() - self.malloc_str_func_addr = rffi.cast(lltype.Signed, - ll_new_str) - if gc_ll_descr.get_funcptr_for_newunicode is not None: - ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() - self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, - ll_new_unicode) - self.memcpy_addr = self.cpu.cast_ptr_to_int(codebuf.memcpy_fn) - self.mc = MachineCodeBlockWrapper(self, self.mc_size, self.cpu.profile_agent) - self._build_failure_recovery(False) - self._build_failure_recovery(True) - if self.cpu.supports_floats: - self._build_failure_recovery(False, withfloats=True) - self._build_failure_recovery(True, withfloats=True) - codebuf.ensure_sse2_floats() - self._build_float_constants() - if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): - self._build_malloc_fixedsize_slowpath() - s = os.environ.get('PYPYLOG') - if s: - if s.find(':') != -1: - s = s.split(':')[-1] - self.set_debug(True) - self._output_loop_log = s + ".count" - # Intialize here instead of __init__ to prevent - # pending_guard_tokens from being considered a prebuilt object, - # which sometimes causes memory leaks since the prebuilt list is - # still considered a GC root after we re-assign - # pending_guard_tokens in write_pending_failure_recoveries - self.pending_guard_tokens = [] + def setup_once(self): + # the address of the function called by 'new' + gc_ll_descr = self.cpu.gc_ll_descr + gc_ll_descr.initialize() + ll_new = gc_ll_descr.get_funcptr_for_new() + self.malloc_func_addr = rffi.cast(lltype.Signed, ll_new) + if gc_ll_descr.get_funcptr_for_newarray is not None: + ll_new_array = gc_ll_descr.get_funcptr_for_newarray() + self.malloc_array_func_addr = rffi.cast(lltype.Signed, + ll_new_array) + if gc_ll_descr.get_funcptr_for_newstr is not None: + ll_new_str = gc_ll_descr.get_funcptr_for_newstr() + self.malloc_str_func_addr = rffi.cast(lltype.Signed, + ll_new_str) + if gc_ll_descr.get_funcptr_for_newunicode is not None: + ll_new_unicode = gc_ll_descr.get_funcptr_for_newunicode() + self.malloc_unicode_func_addr = rffi.cast(lltype.Signed, + ll_new_unicode) + self.memcpy_addr = self.cpu.cast_ptr_to_int(support.memcpy_fn) + self._build_failure_recovery(False) + self._build_failure_recovery(True) + if self.cpu.supports_floats: + self._build_failure_recovery(False, withfloats=True) + self._build_failure_recovery(True, withfloats=True) + support.ensure_sse2_floats() + self._build_float_constants() + if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): + self._build_malloc_fixedsize_slowpath() + debug_start('jit-backend-counts') + self.set_debug(have_debug_prints()) + debug_stop('jit-backend-counts') + + def setup(self, looptoken): + assert self.memcpy_addr != 0, "setup_once() not called?" + self.pending_guard_tokens = [] + self.mc = codebuf.MachineCodeBlockWrapper() + if self.datablockwrapper is None: + allblocks = self.get_asmmemmgr_blocks(looptoken) + self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, + allblocks) + + def teardown(self): + self.pending_guard_tokens = None + self.mc = None + self.looppos = -1 + self.currently_compiling_loop = None def finish_once(self): if self._debug: - output_log = self._output_loop_log - assert output_log is not None - f = open_file_as_stream(output_log, "w") + debug_start('jit-backend-counts') for i in range(len(self.loop_run_counters)): - name, struct = self.loop_run_counters[i] - f.write(str(name) + ":" + str(struct.i) + "\n") - f.close() + struct = self.loop_run_counters[i] + debug_print(str(i) + ':' + str(struct.i)) + debug_stop('jit-backend-counts') def _build_float_constants(self): - # 44 bytes: 32 bytes for the data, and up to 12 bytes for alignment - addr = lltype.malloc(rffi.CArray(lltype.Char), 44, flavor='raw', - track_allocation=False) - if not we_are_translated(): - self._keepalive_malloced_float_consts = addr - float_constants = rffi.cast(lltype.Signed, addr) - float_constants = (float_constants + 15) & ~15 # align to 16 bytes + datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, []) + float_constants = datablockwrapper.malloc_aligned(32, alignment=16) + datablockwrapper.done() addr = rffi.cast(rffi.CArrayPtr(lltype.Char), float_constants) qword_padding = '\x00\x00\x00\x00\x00\x00\x00\x00' # 0x8000000000000000 @@ -271,45 +167,56 @@ def _build_malloc_fixedsize_slowpath(self): # ---------- first helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath1 = self.mc.tell() + mc = codebuf.MachineCodeBlockWrapper() if self.cpu.supports_floats: # save the XMM registers in for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 - self.mc.MOVSD_sx((WORD*2)+8*i, i) - self.mc.SUB_rr(edx.value, eax.value) # compute the size we want + mc.MOVSD_sx((WORD*2)+8*i, i) + mc.SUB_rr(edx.value, eax.value) # compute the size we want if IS_X86_32: - self.mc.MOV_sr(WORD, edx.value) # save it as the new argument + mc.MOV_sr(WORD, edx.value) # save it as the new argument elif IS_X86_64: # rdi can be clobbered: its content was forced to the stack # by _fastpath_malloc(), like all other save_around_call_regs. - self.mc.MOV_rr(edi.value, edx.value) + mc.MOV_rr(edi.value, edx.value) addr = self.cpu.gc_ll_descr.get_malloc_fixedsize_slowpath_addr() - self.mc.JMP(imm(addr)) # tail call to the real malloc + mc.JMP(imm(addr)) # tail call to the real malloc + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.malloc_fixedsize_slowpath1 = rawstart # ---------- second helper for the slow path of malloc ---------- - self.malloc_fixedsize_slowpath2 = self.mc.tell() + mc = codebuf.MachineCodeBlockWrapper() if self.cpu.supports_floats: # restore the XMM registers for i in range(self.cpu.NUM_REGS):# from where they were saved - self.mc.MOVSD_xs(i, (WORD*2)+8*i) + mc.MOVSD_xs(i, (WORD*2)+8*i) nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - self.mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX - self.mc.RET() - self.mc.done() + mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + mc.RET() + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.malloc_fixedsize_slowpath2 = rawstart def assemble_loop(self, inputargs, operations, looptoken, log): - """adds the following attributes to looptoken: + '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) _x86_bootstrap_code (an integer giving an address) - _x86_direct_bootstrap_code + _x86_direct_bootstrap_code ( " " " " ) _x86_frame_depth _x86_param_depth _x86_arglocs _x86_debug_checksum - """ + ''' + # XXX this function is too longish and contains some code + # duplication with assemble_bridge(). Also, we should think + # about not storing on 'self' attributes that will live only + # for the duration of compiling one loop or a one bridge. + + clt = CompiledLoopToken(self.cpu, looptoken.number) + looptoken.compiled_loop_token = clt if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.setup() + self.setup(looptoken) + self.currently_compiling_loop = looptoken funcname = self._find_debug_merge_point(operations) if log: self._register_counter() @@ -319,43 +226,61 @@ arglocs = regalloc.prepare_loop(inputargs, operations, looptoken) looptoken._x86_arglocs = arglocs - # profile support - name = "Loop # %s: %s" % (looptoken.number, funcname) - self.mc.start_function(name) - looptoken._x86_bootstrap_code = self.mc.tell() - adr_stackadjust = self._assemble_bootstrap_code(inputargs, arglocs) - curadr = self.mc.tell() - looptoken._x86_loop_code = curadr + bootstrappos = self.mc.get_relative_pos() + stackadjustpos = self._assemble_bootstrap_code(inputargs, arglocs) + self.looppos = self.mc.get_relative_pos() looptoken._x86_frame_depth = -1 # temporarily looptoken._x86_param_depth = -1 # temporarily frame_depth, param_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) looptoken._x86_frame_depth = frame_depth looptoken._x86_param_depth = param_depth - looptoken._x86_direct_bootstrap_code = self.mc.tell() - self._assemble_bootstrap_direct_call(arglocs, curadr, + directbootstrappos = self.mc.get_relative_pos() + self._assemble_bootstrap_direct_call(arglocs, self.looppos, frame_depth+param_depth) - # - debug_print("Loop #%d has address %x to %x" % (looptoken.number, - looptoken._x86_loop_code, - self.mc.tell())) - self.mc.end_function() self.write_pending_failure_recoveries() - - def assemble_bridge(self, faildescr, inputargs, operations, log): + fullsize = self.mc.get_relative_pos() + # + rawstart = self.materialize_loop(looptoken) + debug_print("Loop #%d (%s) has address %x to %x" % ( + looptoken.number, funcname, + rawstart + self.looppos, + rawstart + directbootstrappos)) + self._patch_stackadjust(rawstart + stackadjustpos, + frame_depth + param_depth) + self.patch_pending_failure_recoveries(rawstart) + # + looptoken._x86_bootstrap_code = rawstart + bootstrappos + looptoken._x86_loop_code = rawstart + self.looppos + looptoken._x86_direct_bootstrap_code = rawstart + directbootstrappos + self.teardown() + # oprofile support + if self.cpu.profile_agent is not None: + name = "Loop # %s: %s" % (looptoken.number, funcname) + self.cpu.profile_agent.native_code_written(name, + rawstart, fullsize) + + def assemble_bridge(self, faildescr, inputargs, operations, + original_loop_token, log): if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - self.setup() + descr_number = self.cpu.get_fail_descr_number(faildescr) + try: + failure_recovery = self._find_failure_recovery_bytecode(faildescr) + except ValueError: + debug_print("Bridge out of guard", descr_number, + "was already compiled!") + return + + self.setup(original_loop_token) funcname = self._find_debug_merge_point(operations) if log: self._register_counter() operations = self._inject_debugging_code(faildescr, operations) - arglocs = self.rebuild_faillocs_from_descr( - faildescr._x86_failure_recovery_bytecode) + arglocs = self.rebuild_faillocs_from_descr(failure_recovery) if not we_are_translated(): assert ([loc.assembler() for loc in arglocs] == [loc.assembler() for loc in faildescr._x86_debug_faillocs]) @@ -364,37 +289,63 @@ regalloc.prepare_bridge(fail_depths, inputargs, arglocs, operations) - # oprofile support - descr_number = self.cpu.get_fail_descr_number(faildescr) - name = "Bridge # %s: %s" % (descr_number, funcname) - self.mc.start_function(name) - - adr_bridge = self.mc.tell() - adr_stackadjust = self._patchable_stackadjust() + stackadjustpos = self._patchable_stackadjust() frame_depth, param_depth = self._assemble(regalloc, operations) - self._patch_stackadjust(adr_stackadjust, frame_depth+param_depth) + codeendpos = self.mc.get_relative_pos() + self.write_pending_failure_recoveries() + fullsize = self.mc.get_relative_pos() + # + rawstart = self.materialize_loop(original_loop_token) + + debug_print("Bridge out of guard %d (%s) has address %x to %x" % + (descr_number, funcname, rawstart, rawstart + codeendpos)) + self._patch_stackadjust(rawstart + stackadjustpos, + frame_depth + param_depth) + self.patch_pending_failure_recoveries(rawstart) if not we_are_translated(): # for the benefit of tests faildescr._x86_bridge_frame_depth = frame_depth faildescr._x86_bridge_param_depth = param_depth # patch the jump from original guard - self.patch_jump_for_descr(faildescr, adr_bridge) - debug_print("Bridge out of guard %d has address %x to %x" % - (descr_number, adr_bridge, self.mc.tell())) - self.mc.end_function() - self.write_pending_failure_recoveries() + self.patch_jump_for_descr(faildescr, rawstart) + self.teardown() + # oprofile support + if self.cpu.profile_agent is not None: + name = "Bridge # %s: %s" % (descr_number, funcname) + self.cpu.profile_agent.native_code_written(name, + rawstart, fullsize) def write_pending_failure_recoveries(self): + # for each pending guard, generate the code of the recovery stub + # at the end of self.mc. for tok in self.pending_guard_tokens: - # Okay to write to _mc because we've already made sure that - # there's enough space by "reserving" bytes. - addr = self.generate_quick_failure(self.mc._mc, tok.faildescr, tok.failargs, tok.fail_locs, tok.exc, tok.desc_bytes) - tok.faildescr._x86_adr_recovery_stub = addr - self.patch_jump_for_descr(tok.faildescr, addr) + tok.pos_recovery_stub = self.generate_quick_failure(tok) - self.pending_guard_tokens = [] - self.mc.reset_reserved_bytes() - self.mc.done() + def patch_pending_failure_recoveries(self, rawstart): + # after we wrote the assembler to raw memory, set up + # tok.faildescr._x86_adr_jump_offset to contain the raw address of + # the 4-byte target field in the JMP/Jcond instruction, and patch + # the field in question to point (initially) to the recovery stub + for tok in self.pending_guard_tokens: + addr = rawstart + tok.pos_jump_offset + tok.faildescr._x86_adr_jump_offset = addr + relative_target = tok.pos_recovery_stub - (tok.pos_jump_offset + 4) + assert rx86.fits_in_32bits(relative_target) + p = rffi.cast(rffi.INTP, addr) + p[0] = rffi.cast(rffi.INT, relative_target) + + def get_asmmemmgr_blocks(self, looptoken): + clt = looptoken.compiled_loop_token + if clt.asmmemmgr_blocks is None: + clt.asmmemmgr_blocks = [] + return clt.asmmemmgr_blocks + + def materialize_loop(self, looptoken): + self.datablockwrapper.done() # finish using cpu.asmmemmgr + self.datablockwrapper = None + allblocks = self.get_asmmemmgr_blocks(looptoken) + return self.mc.materialize(self.cpu.asmmemmgr, allblocks, + self.cpu.gc_ll_descr.gcrootmap) def _find_debug_merge_point(self, operations): @@ -409,29 +360,50 @@ def _register_counter(self): if self._debug: + # YYY very minor leak -- we need the counters to stay alive + # forever, just because we want to report them at the end + # of the process struct = lltype.malloc(DEBUG_COUNTER, flavor='raw', - track_allocation=False) # known to leak + track_allocation=False) struct.i = 0 - self.loop_run_counters.append((len(self.loop_run_counters), struct)) - + self.loop_run_counters.append(struct) + + def _find_failure_recovery_bytecode(self, faildescr): + adr_jump_offset = faildescr._x86_adr_jump_offset + if adr_jump_offset == 0: + raise ValueError + # follow the JMP/Jcond + p = rffi.cast(rffi.INTP, adr_jump_offset) + adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) + # skip the CALL + if WORD == 4: + adr_target += 5 # CALL imm + else: + adr_target += 13 # MOV r11, imm; CALL *r11 + return adr_target + def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset - adr_recovery_stub = faildescr._x86_adr_recovery_stub + assert adr_jump_offset != 0 offset = adr_new_target - (adr_jump_offset + 4) # If the new target fits within a rel32 of the jump, just patch # that. Otherwise, leave the original rel32 to the recovery stub in # place, but clobber the recovery stub with a jump to the real # target. + mc = codebuf.MachineCodeBlockWrapper() if rx86.fits_in_32bits(offset): - mc = codebuf.InMemoryCodeBuilder(adr_jump_offset, adr_jump_offset + 4) mc.writeimm32(offset) + mc.copy_to_raw_memory(adr_jump_offset) else: - # "mov r11, addr; jmp r11" is 13 bytes - mc = codebuf.InMemoryCodeBuilder(adr_recovery_stub, adr_recovery_stub + 13) + # "mov r11, addr; jmp r11" is 13 bytes, which fits in there + # because we always write "mov r11, addr; call *r11" in the + # first place. mc.MOV_ri(X86_64_SCRATCH_REG.value, adr_new_target) mc.JMP_r(X86_64_SCRATCH_REG.value) - - mc.done() + p = rffi.cast(rffi.INTP, adr_jump_offset) + adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) + mc.copy_to_raw_memory(adr_target) + faildescr._x86_adr_jump_offset = 0 # means "patched" @specialize.argtype(1) def _inject_debugging_code(self, looptoken, operations): @@ -442,7 +414,7 @@ s += op.getopnum() looptoken._x86_debug_checksum = s c_adr = ConstInt(rffi.cast(lltype.Signed, - self.loop_run_counters[-1][1])) + self.loop_run_counters[-1])) box = BoxInt() box2 = BoxInt() ops = [ResOperation(rop.GETFIELD_RAW, [c_adr], @@ -451,19 +423,11 @@ ResOperation(rop.SETFIELD_RAW, [c_adr, box2], None, descr=self.debug_counter_descr)] operations = ops + operations - # # we need one register free (a bit of a hack, but whatever) - # self.mc.PUSH(eax) - # adr = rffi.cast(lltype.Signed, self.loop_run_counters[-1][1]) - # self.mc.MOV(eax, heap(adr)) - # self.mc.ADD(eax, imm1) - # self.mc.MOV(heap(adr), eax) - # self.mc.POP(eax) return operations def _assemble(self, regalloc, operations): self._regalloc = regalloc regalloc.walk_operations(operations) - self.mc.done() if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.fm.frame_depth @@ -479,30 +443,31 @@ def _patchable_stackadjust(self): # stack adjustment LEA self.mc.LEA32_rb(esp.value, 0) - return self.mc.tell() - 4 + return self.mc.get_relative_pos() - 4 - def _patch_stackadjust(self, adr_lea, reserved_depth): + def _patch_stackadjust(self, adr_lea, allocated_depth): # patch stack adjustment LEA - mc = codebuf.InMemoryCodeBuilder(adr_lea, adr_lea + 4) - # Compute the correct offset for the instruction LEA ESP, [EBP-4*words]. + mc = codebuf.MachineCodeBlockWrapper() + # Compute the correct offset for the instruction LEA ESP, [EBP-4*words] + mc.writeimm32(self._get_offset_of_ebp_from_esp(allocated_depth)) + mc.copy_to_raw_memory(adr_lea) + + def _get_offset_of_ebp_from_esp(self, allocated_depth): # Given that [EBP] is where we saved EBP, i.e. in the last word # of our fixed frame, then the 'words' value is: - words = (self.cpu.FRAME_FIXED_SIZE - 1) + reserved_depth - # align, e.g. for Mac OS X + words = (self.cpu.FRAME_FIXED_SIZE - 1) + allocated_depth + # align, e.g. for Mac OS X aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - mc.writeimm32(-WORD * aligned_words) - mc.done() + return -WORD * aligned_words def _call_header(self): + # NB. the shape of the frame is hard-coded in get_basic_shape() too. + # Also, make sure this is consistent with FRAME_FIXED_SIZE. self.mc.PUSH_r(ebp.value) self.mc.MOV_rr(ebp.value, esp.value) for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) - # NB. the shape of the frame is hard-coded in get_basic_shape() too. - # Also, make sure this is consistent with FRAME_FIXED_SIZE. - return self._patchable_stackadjust() - def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -512,17 +477,16 @@ self.mc.POP_r(ebp.value) self.mc.RET() - def _assemble_bootstrap_direct_call(self, arglocs, jmpadr, stackdepth): + def _assemble_bootstrap_direct_call(self, arglocs, jmppos, stackdepth): if IS_X86_64: - return self._assemble_bootstrap_direct_call_64(arglocs, jmpadr, stackdepth) + return self._assemble_bootstrap_direct_call_64(arglocs, jmppos, stackdepth) # XXX pushing ebx esi and edi is a bit pointless, since we store # all regsiters anyway, for the case of guard_not_forced # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - # XXX not to repeat the logic, a bit around - adr_stackadjust = self._call_header() - self._patch_stackadjust(adr_stackadjust, stackdepth) + self._call_header() + self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if isinstance(loc, RegLoc): @@ -544,9 +508,11 @@ self.mc.MOVSD_xb(xmmtmp.value, (1 + i) * 2 * WORD) assert isinstance(loc, StackLoc) self.mc.MOVSD_bx(loc.value, xmmtmp.value) - self.mc.JMP_l(jmpadr) + endpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(jmppos - endpos) + assert endpos == self.mc.get_relative_pos() - def _assemble_bootstrap_direct_call_64(self, arglocs, jmpadr, stackdepth): + def _assemble_bootstrap_direct_call_64(self, arglocs, jmppos, stackdepth): # XXX: Very similar to _emit_call_64 src_locs = [] @@ -560,8 +526,8 @@ unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - adr_stackadjust = self._call_header() - self._patch_stackadjust(adr_stackadjust, stackdepth) + self._call_header() + self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones assert len(nonfloatlocs) == len(floatlocs) @@ -597,8 +563,9 @@ # clobber the scratch register self.mc.MOV(loc, X86_64_SCRATCH_REG) - finaljmp = self.mc.tell() - self.mc.JMP(imm(jmpadr)) + endpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(jmppos - endpos) + assert endpos == self.mc.get_relative_pos() def redirect_call_assembler(self, oldlooptoken, newlooptoken): # some minimal sanity checking @@ -611,16 +578,17 @@ # Ideally we should rather patch all existing CALLs, but well. oldadr = oldlooptoken._x86_direct_bootstrap_code target = newlooptoken._x86_direct_bootstrap_code - mc = codebuf.InMemoryCodeBuilder(oldadr, oldadr + 16) + mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) - mc.done() + mc.copy_to_raw_memory(oldadr) def _assemble_bootstrap_code(self, inputargs, arglocs): nonfloatlocs, floatlocs = arglocs - adr_stackadjust = self._call_header() + self._call_header() + stackadjustpos = self._patchable_stackadjust() tmp = X86RegisterManager.all_regs[0] xmmtmp = X86XMMRegisterManager.all_regs[0] - self.mc._mc.begin_reuse_scratch_register() + self.mc.begin_reuse_scratch_register() for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] if loc is None: @@ -652,8 +620,8 @@ self.mc.MOVSD(xmmtmp, heap(adr)) assert isinstance(loc, StackLoc) self.mc.MOVSD_bx(loc.value, xmmtmp.value) - self.mc._mc.end_reuse_scratch_register() - return adr_stackadjust + self.mc.end_reuse_scratch_register() + return stackadjustpos def dump(self, text): if not self.verbose: @@ -661,7 +629,8 @@ _prev = Box._extended_display try: Box._extended_display = False - print >> sys.stderr, ' 0x%x %s' % (fixid(self.mc.tell()), text) + pos = self.mc.get_relative_pos() + print >> sys.stderr, ' 0x%x %s' % (pos, text) finally: Box._extended_display = _prev @@ -721,7 +690,7 @@ arglocs, resloc) if not we_are_translated(): # must be added by the genop_guard_list[]() - assert hasattr(faildescr, '_x86_adr_jump_offset') + assert guard_token is self.pending_guard_tokens[-1] def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc, current_depths): @@ -794,9 +763,6 @@ result_loc): guard_opnum = guard_op.getopnum() self.mc.UCOMISD(arglocs[0], arglocs[1]) - # 16 is enough space for the rel8 jumps below and the rel32 - # jump in implement_guard - self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_FALSE: if need_jp: self.mc.J_il8(rx86.Conditions['P'], 6) @@ -964,9 +930,6 @@ def genop_guard_float_ne(self, op, guard_op, guard_token, arglocs, result_loc): guard_opnum = guard_op.getopnum() self.mc.UCOMISD(arglocs[0], arglocs[1]) - # 16 is enough space for the rel8 jumps below and the rel32 - # jump in implement_guard - self.mc.ensure_bytes_available(16 + guard_token.recovery_stub_size()) if guard_opnum == rop.GUARD_TRUE: self.mc.J_il8(rx86.Conditions['P'], 6) self.implement_guard(guard_token, 'E') @@ -1290,13 +1253,11 @@ self.mc.CMP32_mi((locs[0].value, 0), expected_typeid) def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2): - self.mc.ensure_bytes_available(256) self._cmp_guard_class(locs) self.implement_guard(guard_token, 'NE') def genop_guard_guard_nonnull_class(self, ign_1, guard_op, guard_token, locs, ign_2): - self.mc.ensure_bytes_available(256) self.mc.CMP(locs[0], imm1) # Patched below self.mc.J_il8(rx86.Conditions['B'], 0) @@ -1305,7 +1266,7 @@ # patch the JB above offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 - self.mc.overwrite(jb_location-1, [chr(offset)]) + self.mc.overwrite(jb_location-1, chr(offset)) # self.implement_guard(guard_token, 'NE') @@ -1314,42 +1275,33 @@ exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) - desc_bytes = self.failure_recovery_description(failargs, fail_locs) - return GuardToken(faildescr, failargs, fail_locs, exc, desc_bytes) + return GuardToken(faildescr, failargs, fail_locs, exc) - def generate_quick_failure(self, mc, faildescr, failargs, fail_locs, exc, desc_bytes): + def generate_quick_failure(self, guardtok): """Generate the initial code for handling a failure. We try to - keep it as compact as possible. The idea is that this code is - executed at most once (and very often, zero times); when - executed, it generates a more complete piece of code which can - really handle recovery from this particular failure. + keep it as compact as possible. """ - fail_index = self.cpu.get_fail_descr_number(faildescr) - addr = mc.tell() + fail_index = self.cpu.get_fail_descr_number(guardtok.faildescr) + mc = self.mc + startpos = mc.get_relative_pos() withfloats = False - for box in failargs: + for box in guardtok.failargs: if box is not None and box.type == FLOAT: withfloats = True break + exc = guardtok.exc mc.CALL(imm(self.failure_recovery_code[exc + 2 * withfloats])) # write tight data that describes the failure recovery - faildescr._x86_failure_recovery_bytecode = mc.tell() - for byte in desc_bytes: - mc.writechr(ord(byte)) + self.write_failure_recovery_description(mc, guardtok.failargs, + guardtok.fail_locs) # write the fail_index too mc.writeimm32(fail_index) # for testing the decoding, write a final byte 0xCC if not we_are_translated(): - mc.writechr(0xCC) - faildescr._x86_debug_faillocs = [loc for loc in fail_locs - if loc is not None] - - # Make sure the recovery stub is at least 16 bytes long (for the - # case where we overwrite the recovery stub with a 64-bit absolute - # jump) - while mc.tell() - addr < 16: - mc.writechr(0x00) - return addr + mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.fail_locs if loc is not None] + guardtok.faildescr._x86_debug_faillocs = faillocs + return startpos DESCR_REF = 0x00 DESCR_INT = 0x01 @@ -1360,8 +1312,7 @@ CODE_STOP = 0 | DESCR_SPECIAL CODE_HOLE = 4 | DESCR_SPECIAL - def failure_recovery_description(self, failargs, locs): - desc_bytes = [] + def write_failure_recovery_description(self, mc, failargs, locs): for i in range(len(failargs)): arg = failargs[i] if arg is not None: @@ -1381,19 +1332,14 @@ n = loc.value n = kind + 4*n while n > 0x7F: - desc_bytes.append(chr((n & 0x7F) | 0x80)) + mc.writechar(chr((n & 0x7F) | 0x80)) n >>= 7 else: n = self.CODE_HOLE - desc_bytes.append(chr(n)) - desc_bytes.append(chr(self.CODE_STOP)) + mc.writechar(chr(n)) + mc.writechar(chr(self.CODE_STOP)) # assert that the fail_boxes lists are big enough assert len(failargs) <= self.fail_boxes_int.SIZE - return desc_bytes - - def write_failure_recovery_description(self, mc, failargs, locs): - for byte in self.failure_recovery_description(failargs, locs): - mc.writechr(ord(byte)) def rebuild_faillocs_from_descr(self, bytecode): from pypy.jit.backend.x86.regalloc import X86FrameManager @@ -1531,10 +1477,8 @@ self.failure_recovery_func) failure_recovery_func = rffi.cast(lltype.Signed, failure_recovery_func) - mc = self.mc._mc - # Assume that we are called at the beginning, when there is no risk - # that 'mc' runs out of space. Checked by asserts in mc.write(). - recovery_addr = mc.tell() + mc = codebuf.MachineCodeBlockWrapper() + self.mc = mc # Push all general purpose registers for gpr in range(self.cpu.NUM_REGS-1, -1, -1): @@ -1585,11 +1529,12 @@ # above. self._call_footer() - self.mc.done() - self.failure_recovery_code[exc + 2 * withfloats] = recovery_addr + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None def generate_failure(self, fail_index, locs, exc, locs_are_ref): - self.mc._mc.begin_reuse_scratch_register() + self.mc.begin_reuse_scratch_register() for i in range(len(locs)): loc = locs[i] if isinstance(loc, RegLoc): @@ -1616,7 +1561,7 @@ adr = self.fail_boxes_int.get_addr_for_num(i) self.mc.MOV(eax, loc) self.mc.MOV(heap(adr), eax) - self.mc._mc.end_reuse_scratch_register() + self.mc.end_reuse_scratch_register() # we call a provided function that will # - call our on_leave_jitted_hook which will mark @@ -1632,17 +1577,13 @@ self._call_footer() def implement_guard(self, guard_token, condition=None): - self.mc.reserve_bytes(guard_token.recovery_stub_size()) - self.pending_guard_tokens.append(guard_token) - # These jumps are patched later, the mc.tell() are just - # dummy values. Also, use self.mc._mc to avoid triggering a - # "buffer full" exactly here. - mc = self.mc._mc + # These jumps are patched later. if condition: - mc.J_il(rx86.Conditions[condition], mc.tell()) + self.mc.J_il(rx86.Conditions[condition], 0) else: - mc.JMP_l(mc.tell()) - guard_token.faildescr._x86_adr_jump_offset = mc.tell() - 4 + self.mc.JMP_l(0) + guard_token.pos_jump_offset = self.mc.get_relative_pos() - 4 + self.pending_guard_tokens.append(guard_token) def genop_call(self, op, arglocs, resloc): sizeloc = arglocs[0] @@ -1697,7 +1638,6 @@ # Write a call to the direct_bootstrap_code of the target assembler self._emit_call(imm(descr._x86_direct_bootstrap_code), arglocs, 2, tmp=eax) - self.mc.ensure_bytes_available(256) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -1733,7 +1673,7 @@ # Path B: fast path. Must load the return value, and reset the token offset = jmp_location - je_location assert 0 < offset <= 127 - self.mc.overwrite(je_location - 1, [chr(offset)]) + self.mc.overwrite(je_location - 1, chr(offset)) # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: @@ -1768,7 +1708,7 @@ # Here we join Path A and Path B again offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 - self.mc.overwrite(jmp_location - 1, [chr(offset)]) + self.mc.overwrite(jmp_location - 1, chr(offset)) self.mc.CMP_bi(FORCE_INDEX_OFS, 0) self.implement_guard(guard_token, 'L') @@ -1783,9 +1723,6 @@ cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) loc_base = arglocs[0] - # ensure that enough bytes are available to write the whole - # following piece of code atomically (for the JZ) - self.mc.ensure_bytes_available(256) self.mc.TEST8_mi((loc_base.value, descr.jit_wb_if_flag_byteofs), descr.jit_wb_if_flag_singlebyte) self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later @@ -1828,7 +1765,7 @@ # patch the JZ above offset = self.mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - self.mc.overwrite(jz_location-1, [chr(offset)]) + self.mc.overwrite(jz_location-1, chr(offset)) def genop_force_token(self, op, arglocs, resloc): # RegAlloc.consider_force_token ensures this: @@ -1851,18 +1788,21 @@ gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: mark = self._regalloc.get_mark_gc_roots(gcrootmap) - gcrootmap.put(rffi.cast(llmemory.Address, self.mc.tell()), mark) + self.mc.insert_gcroot_marker(mark) def target_arglocs(self, loop_token): return loop_token._x86_arglocs def closing_jump(self, loop_token): - self.mc.JMP(imm(loop_token._x86_loop_code)) + if loop_token is self.currently_compiling_loop: + curpos = self.mc.get_relative_pos() + 5 + self.mc.JMP_l(self.looppos - curpos) + else: + self.mc.JMP(imm(loop_token._x86_loop_code)) def malloc_cond_fixedsize(self, nursery_free_adr, nursery_top_adr, size, tid): size = max(size, self.cpu.gc_ll_descr.minimal_size_in_nursery) - self.mc.ensure_bytes_available(256) self.mc.MOV(eax, heap(nursery_free_adr)) self.mc.LEA_rm(edx.value, (eax.value, size)) self.mc.CMP(edx, heap(nursery_top_adr)) @@ -1890,7 +1830,7 @@ offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 - self.mc.overwrite(jmp_adr-1, [chr(offset)]) + self.mc.overwrite(jmp_adr-1, chr(offset)) # on 64-bits, 'tid' is a value that fits in 31 bits self.mc.MOV_mi((eax.value, 0), tid) self.mc.MOV(heap(nursery_free_adr), edx) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/regalloc.py Sat Dec 11 15:10:15 2010 @@ -60,32 +60,6 @@ r15: 5, } -class FloatConstants(object): - BASE_CONSTANT_SIZE = 1000 - - def __init__(self): - self.cur_array_free = 0 - self.const_id = 0 - - def _get_new_array(self): - n = self.BASE_CONSTANT_SIZE - # known to leak - self.cur_array = lltype.malloc(rffi.CArray(lltype.Float), n, - flavor='raw', track_allocation=False) - self.cur_array_free = n - _get_new_array._dont_inline_ = True - - def record_float(self, floatval): - if self.cur_array_free == 0: - self._get_new_array() - arr = self.cur_array - n = self.cur_array_free - 1 - arr[n] = floatval - self.cur_array_free = n - self.const_id += 1 - return (self.const_id, rffi.cast(lltype.Signed, arr) + n * 8) - - class X86XMMRegisterManager(RegisterManager): box_types = [FLOAT] @@ -93,20 +67,11 @@ # we never need lower byte I hope save_around_call_regs = all_regs - def __init__(self, longevity, frame_manager=None, assembler=None): - RegisterManager.__init__(self, longevity, frame_manager=frame_manager, - assembler=assembler) - if assembler is None: - self.float_constants = FloatConstants() - else: - if assembler._float_constants is None: - assembler._float_constants = FloatConstants() - self.float_constants = assembler._float_constants - def convert_to_imm(self, c): - const_id, adr = self.float_constants.record_float(c.getfloat()) - return ConstFloatLoc(adr, const_id) - + adr = self.assembler.datablockwrapper.malloc_aligned(8, 8) + rffi.cast(rffi.CArrayPtr(rffi.DOUBLE), adr)[0] = c.getfloat() + return ConstFloatLoc(adr) + def after_call(self, v): # the result is stored in st0, but we don't have this around, # so genop_call will move it to some frame location immediately @@ -320,11 +285,22 @@ def locs_for_fail(self, guard_op): return [self.loc(v) for v in guard_op.getfailargs()] + def get_current_depth(self): + # return (self.fm.frame_depth, self.param_depth), but trying to share + # the resulting tuple among several calls + arg0 = self.fm.frame_depth + arg1 = self.param_depth + result = self.assembler._current_depths_cache + if result[0] != arg0 or result[1] != arg1: + result = (arg0, arg1) + self.assembler._current_depths_cache = result + return result + def perform_with_guard(self, op, guard_op, arglocs, result_loc): faillocs = self.locs_for_fail(guard_op) self.rm.position += 1 self.xrm.position += 1 - current_depths = (self.fm.frame_depth, self.param_depth) + current_depths = self.get_current_depth() self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc, current_depths) @@ -340,7 +316,7 @@ arglocs)) else: self.assembler.dump('%s(%s)' % (guard_op, arglocs)) - current_depths = (self.fm.frame_depth, self.param_depth) + current_depths = self.get_current_depth() self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, result_loc, current_depths) @@ -1097,7 +1073,8 @@ if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): assert reg in self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX gcrootmap.add_callee_save_reg(shape, self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX[reg]) - return gcrootmap.compress_callshape(shape) + return gcrootmap.compress_callshape(shape, + self.assembler.datablockwrapper) def consider_force_token(self, op): loc = self.rm.force_allocate_reg(op.result) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/regloc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/regloc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/regloc.py Sat Dec 11 15:10:15 2010 @@ -177,24 +177,15 @@ class ConstFloatLoc(AssemblerLocation): # XXX: We have to use this class instead of just AddressLoc because - # AddressLoc is "untyped" and also we to have need some sort of unique - # identifier that we can use in _getregkey (for jump.py) - + # we want a width of 8 (... I think. Check this!) _immutable_ = True - width = 8 - def __init__(self, address, const_id): + def __init__(self, address): self.value = address - self.const_id = const_id def __repr__(self): - return '' % (self.value, self.const_id) - - def _getregkey(self): - # XXX: 1000 is kind of magic: We just don't want to be confused - # with any registers - return 1000 + self.const_id + return '' % (self.value,) def location_code(self): return 'j' @@ -334,9 +325,9 @@ if code == possible_code: val = getattr(loc, "value_" + possible_code)() if possible_code == 'i': - offset = intmask(val - (self.tell() + 5)) - if rx86.fits_in_32bits(offset): + if self.WORD == 4: _rx86_getattr(self, name + "_l")(val) + self.add_pending_relocation() else: assert self.WORD == 8 self._load_scratch(val) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/runner.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/runner.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/runner.py Sat Dec 11 15:10:15 2010 @@ -44,6 +44,7 @@ def setup_once(self): self.profile_agent.startup() + self.assembler.setup_once() def finish_once(self): self.assembler.finish_once() @@ -53,9 +54,12 @@ self.assembler.assemble_loop(inputargs, operations, looptoken, log=log) - def compile_bridge(self, faildescr, inputargs, operations, log=True): + def compile_bridge(self, faildescr, inputargs, operations, + original_loop_token, log=True): + clt = original_loop_token.compiled_loop_token + clt.compiling_a_bridge() self.assembler.assemble_bridge(faildescr, inputargs, operations, - log=log) + original_loop_token, log=log) def set_future_value_int(self, index, intvalue): self.assembler.fail_boxes_int.setitem(index, intvalue) @@ -124,8 +128,8 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = -1 - bytecode = rffi.cast(rffi.UCHARP, - faildescr._x86_failure_recovery_bytecode) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) # start of "no gc operation!" block fail_index_2 = self.assembler.grab_frame_values( bytecode, @@ -165,17 +169,12 @@ CPU = CPU386 # silence warnings - -history.LoopToken._x86_param_depth = 0 -history.LoopToken._x86_arglocs = (None, None) -history.LoopToken._x86_frame_depth = 0 -history.LoopToken._x86_bootstrap_code = 0 -history.LoopToken._x86_direct_bootstrap_code = 0 -history.LoopToken._x86_failure_recovery_bytecode = 0 -history.LoopToken._x86_loop_code = 0 -history.LoopToken._x86_current_depths = (0, 0) - -compile._DoneWithThisFrameDescr._x86_current_depths = (0, 0) -compile._DoneWithThisFrameDescr._x86_failure_recovery_bytecode = 0 -compile._DoneWithThisFrameDescr._x86_adr_jump_offset = 0 - +##history.LoopToken._x86_param_depth = 0 +##history.LoopToken._x86_arglocs = (None, None) +##history.LoopToken._x86_frame_depth = 0 +##history.LoopToken._x86_bootstrap_code = 0 +##history.LoopToken._x86_direct_bootstrap_code = 0 +##history.LoopToken._x86_loop_code = 0 +##history.LoopToken._x86_debug_checksum = 0 +##compile.AbstractFailDescr._x86_current_depths = (0, 0) +##compile.AbstractFailDescr._x86_adr_jump_offset = 0 Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/rx86.py Sat Dec 11 15:10:15 2010 @@ -137,10 +137,9 @@ # ____________________________________________________________ # Emit an immediate displacement (relative to the cur insn) -def encode_relative(mc, target, _, orbyte): +def encode_relative(mc, relative_target, _, orbyte): assert orbyte == 0 - offset = intmask(target - (mc.tell() + 4)) - mc.writeimm32(offset) + mc.writeimm32(relative_target) return 0 def relative(argnum): Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/support.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/support.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/support.py Sat Dec 11 15:10:15 2010 @@ -1,4 +1,7 @@ +import sys from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from pypy.translator.tool.cbuild import ExternalCompilationInfo + def values_array(TP, size): ATP = lltype.GcArray(TP) @@ -23,3 +26,22 @@ return True return ValuesArray() + +# ____________________________________________________________ + +memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) + +# ____________________________________________________________ + +if sys.platform == 'win32': + ensure_sse2_floats = lambda : None +else: + _sse2_eci = ExternalCompilationInfo( + compile_extra = ['-msse2', '-mfpmath=sse'], + separate_module_sources = ['void PYPY_NO_OP(void) {}'], + ) + ensure_sse2_floats = rffi.llexternal('PYPY_NO_OP', [], lltype.Void, + compilation_info=_sse2_eci, + sandboxsafe=True) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_assembler.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_assembler.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_assembler.py Sat Dec 11 15:10:15 2010 @@ -1,5 +1,5 @@ from pypy.jit.backend.x86.regloc import * -from pypy.jit.backend.x86.assembler import Assembler386, MachineCodeBlockWrapper +from pypy.jit.backend.x86.assembler import Assembler386 from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, INT, REF, FLOAT from pypy.rlib.rarithmetic import intmask @@ -19,28 +19,10 @@ return 42 class FakeMC: - def __init__(self, base_address=0): + def __init__(self): self.content = [] - self._size = 100 - self.base_address = base_address - def writechr(self, n): - self.content.append(n) - def tell(self): - return self.base_address + len(self.content) - def get_relative_pos(self): - return len(self.content) - def JMP(self, *args): - self.content.append(("JMP", args)) - def done(self): - pass - def PUSH_r(self, reg): - pass - def POP_r(self, reg): - pass - -class FakeAssembler: - def write_pending_failure_recoveries(self): - pass + def writechar(self, char): + self.content.append(ord(char)) def test_write_failure_recovery_description(): assembler = Assembler386(FakeCPU()) @@ -255,41 +237,3 @@ assert assembler.fail_boxes_int.getitem(i) == expected_ints[i] assert assembler.fail_boxes_ptr.getitem(i) == expected_ptrs[i] assert assembler.fail_boxes_float.getitem(i) == expected_floats[i] - -class FakeProfileAgent(object): - def __init__(self): - self.functions = [] - - def native_code_written(self, name, address, size): - self.functions.append((name, address, size)) - -class FakeMCWrapper(MachineCodeBlockWrapper): - count = 0 - def _instantiate_mc(self): - self.count += 1 - return FakeMC(200 * (self.count - 1)) - -def test_mc_wrapper_profile_agent(): - agent = FakeProfileAgent() - assembler = FakeAssembler() - mc = FakeMCWrapper(assembler, 100, agent) - mc.start_function("abc") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4)] - mc.writechr("x") - mc.start_function("cde") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4), ("cde", 5, 4)] - mc.start_function("xyz") - for i in range(50): - mc.writechr("x") - mc.end_function() - assert agent.functions == [("abc", 0, 4), ("cde", 5, 4), ("xyz", 9, 29), ("xyz", 200, 22)] Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_gc_integration.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_gc_integration.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_gc_integration.py Sat Dec 11 15:10:15 2010 @@ -33,7 +33,8 @@ def add_callee_save_reg(self, shape, reg_index): index_to_name = { 1: 'ebx', 2: 'esi', 3: 'edi' } shape.append(index_to_name[reg_index]) - def compress_callshape(self, shape): + def compress_callshape(self, shape, datablockwrapper): + assert datablockwrapper == 'fakedatablockwrapper' assert shape[0] == 'shape' return ['compressed'] + shape[1:] @@ -58,7 +59,9 @@ def test_mark_gc_roots(self): cpu = CPU(None, None) + cpu.setup_once() regalloc = RegAlloc(MockAssembler(cpu, MockGcDescr(False))) + regalloc.assembler.datablockwrapper = 'fakedatablockwrapper' boxes = [BoxPtr() for i in range(len(X86RegisterManager.all_regs))] longevity = {} for box in boxes: @@ -90,6 +93,7 @@ cpu = CPU(None, None) cpu.gc_ll_descr = MockGcDescr(False) + cpu.setup_once() S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', ('field', lltype.Ptr(S)), @@ -214,6 +218,7 @@ cpu = CPU(None, None) cpu.vtable_offset = WORD cpu.gc_ll_descr = GCDescrFastpathMalloc() + cpu.setup_once() NODE = lltype.Struct('node', ('tid', lltype.Signed), ('value', lltype.Signed)) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regalloc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regalloc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regalloc.py Sat Dec 11 15:10:15 2010 @@ -9,7 +9,7 @@ from pypy.jit.backend.llsupport.descr import GcCache from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.regalloc import RegAlloc, X86RegisterManager,\ - FloatConstants, is_comparison_or_ovf_op + is_comparison_or_ovf_op from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -41,7 +41,10 @@ self.movs = [] self.performs = [] self.lea = [] - self.cpu = cpu or CPU(None, None) + if cpu is None: + cpu = CPU(None, None) + cpu.setup_once() + self.cpu = cpu if gc_ll_descr is None: gc_ll_descr = MockGcDescr(False) self.cpu.gc_ll_descr = gc_ll_descr @@ -76,6 +79,7 @@ class BaseTestRegalloc(object): cpu = CPU(None, None) + cpu.setup_once() def raising_func(i): if i: @@ -166,7 +170,8 @@ assert ([box.type for box in bridge.inputargs] == [box.type for box in guard_op.getfailargs()]) faildescr = guard_op.getdescr() - self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations) + self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations, + loop.token) return bridge def run(self, loop): @@ -515,16 +520,6 @@ self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] - def test_float_overflow_const_list(self): - ops = ['[f0]'] - BASE_CONSTANT_SIZE = FloatConstants.BASE_CONSTANT_SIZE - for i in range(BASE_CONSTANT_SIZE * 2): - ops.append('f%d = float_add(f%d, 3.5)' % (i + 1, i)) - ops.append('finish(f%d)' % (BASE_CONSTANT_SIZE * 2)) - ops = "\n".join(ops) - self.interpret(ops, [0.1]) - assert abs(self.getfloat(0) - (BASE_CONSTANT_SIZE * 2) * 3.5 - 0.1) < 0.00001 - def test_lt_const(self): ops = ''' [f0] Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regalloc2.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regalloc2.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regalloc2.py Sat Dec 11 15:10:15 2010 @@ -19,6 +19,7 @@ ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, 9) @@ -41,6 +42,7 @@ ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, -10) @@ -137,6 +139,7 @@ ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, -13) @@ -251,6 +254,7 @@ ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), ] cpu = CPU(None, None) + cpu.setup_once() looptoken = LoopToken() cpu.compile_loop(inputargs, operations, looptoken) cpu.set_future_value_int(0, 17) Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regloc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regloc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_regloc.py Sat Dec 11 15:10:15 2010 @@ -1,4 +1,4 @@ -import struct +import struct, sys from pypy.jit.backend.x86.regloc import * from pypy.jit.backend.x86.test.test_rx86 import CodeBuilder32, CodeBuilder64, assert_encodes_as from pypy.jit.backend.x86.assembler import heap @@ -37,26 +37,36 @@ assert_encodes_as(cb64, "CMP16", (ecx, ImmedLoc(12345)), '\x66\x81\xF9\x39\x30') assert_encodes_as(cb64, "CMP16", (AddressLoc(r13, ImmedLoc(0), 0, 0), ImmedLoc(12345)), '\x66\x41\x81\x7D\x00\x39\x30') -def test_jmp_wraparound(): - if not IS_X86_32: - py.test.skip() - - pos_addr = intmask(0x7FFFFF00) - neg_addr = intmask(0x800000BB) - - # JMP to "negative" address from "positive" address - s = cb32() - s.base_address = pos_addr - s.JMP(ImmedLoc(neg_addr)) - expected_ofs = neg_addr - (pos_addr+5) - assert s.getvalue() == '\xE9' + struct.pack(" sys.maxint: + continue + mc = codebuf.MachineCodeBlockWrapper() + mc.CALL(ImmedLoc(target)) + length = mc.get_relative_pos() + buf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw') + rawstart = rffi.cast(lltype.Signed, buf) + if IS_X86_32: + assert length == 5 + assert mc.relocations == [5] + expected = "\xE8" + struct.pack('= 40 # randomish number @@ -370,7 +376,7 @@ ] bridge[1].setfailargs([i1b]) - self.cpu.compile_bridge(faildescr1, [i1b], bridge) + self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken) name, address, size = agent.functions[1] assert name == "Bridge # 0: bye" # Would be exactly ==, but there are some guard failure recovery @@ -397,104 +403,15 @@ assert res.value == 4.0 -class TestX86OverflowMC(TestX86): - - def setup_method(self, meth): - self.cpu = CPU(rtyper=None, stats=FakeStats()) - self.cpu.assembler.mc_size = 1024 - - def test_overflow_mc(self): - ops = [] - base_v = BoxInt() - v = base_v - for i in range(1024): - next_v = BoxInt() - ops.append(ResOperation(rop.INT_ADD, [v, ConstInt(1)], next_v)) - v = next_v - ops.append(ResOperation(rop.FINISH, [v], None, - descr=BasicFailDescr())) - looptoken = LoopToken() - self.cpu.assembler.setup() - old_mc_mc = self.cpu.assembler.mc._mc - self.cpu.compile_loop([base_v], ops, looptoken) - assert self.cpu.assembler.mc._mc != old_mc_mc # overflowed - self.cpu.set_future_value_int(0, base_v.value) - self.cpu.execute_token(looptoken) - assert self.cpu.get_latest_value_int(0) == 1024 - - def test_overflow_guard_float_cmp(self): - # The float comparisons on x86 tend to use small relative jumps, - # which may run into trouble if they fall on the edge of a - # MachineCodeBlock change. - a = BoxFloat(1.0) - b = BoxFloat(2.0) - failed = BoxInt(41) - finished = BoxInt(42) - - # We select guards that will always succeed, so that execution will - # continue through the entire set of comparisions - ops_to_test = ( - (rop.FLOAT_LT, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_LT, [b, a], rop.GUARD_FALSE), - - (rop.FLOAT_LE, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_LE, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_LE, [b, a], rop.GUARD_FALSE), - - (rop.FLOAT_EQ, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_EQ, [a, b], rop.GUARD_FALSE), - - (rop.FLOAT_NE, [a, b], rop.GUARD_TRUE), - (rop.FLOAT_NE, [a, a], rop.GUARD_FALSE), - - (rop.FLOAT_GT, [b, a], rop.GUARD_TRUE), - (rop.FLOAT_GT, [a, b], rop.GUARD_FALSE), - - (rop.FLOAT_GE, [a, a], rop.GUARD_TRUE), - (rop.FLOAT_GE, [b, a], rop.GUARD_TRUE), - (rop.FLOAT_GE, [a, b], rop.GUARD_FALSE), - ) - - for float_op, args, guard_op in ops_to_test: - ops = [] - - for i in range(200): - cmp_result = BoxInt() - ops.append(ResOperation(float_op, args, cmp_result)) - ops.append(ResOperation(guard_op, [cmp_result], None, descr=BasicFailDescr())) - ops[-1].setfailargs([failed]) - - ops.append(ResOperation(rop.FINISH, [finished], None, descr=BasicFailDescr())) - - looptoken = LoopToken() - self.cpu.compile_loop([a, b, failed, finished], ops, looptoken) - self.cpu.set_future_value_float(0, a.value) - self.cpu.set_future_value_float(1, b.value) - self.cpu.set_future_value_int(2, failed.value) - self.cpu.set_future_value_int(3, finished.value) - self.cpu.execute_token(looptoken) - - # Really just a sanity check. We're actually interested in - # whether the test segfaults. - assert self.cpu.get_latest_value_int(0) == finished.value - - def test_overflow_guard_exception(self): - for i in range(50): - self.test_exceptions() - - class TestDebuggingAssembler(object): def setup_method(self, meth): - self.pypylog = os.environ.get('PYPYLOG', None) - self.logfile = str(udir.join('x86_runner.log')) - os.environ['PYPYLOG'] = "mumble:" + self.logfile self.cpu = CPU(rtyper=None, stats=FakeStats()) - - def teardown_method(self, meth): - if self.pypylog is not None: - os.environ['PYPYLOG'] = self.pypylog + self.cpu.setup_once() def test_debugger_on(self): + from pypy.tool.logparser import parse_log_file, extract_category + from pypy.rlib import debug + loop = """ [i0] debug_merge_point('xyz', 0) @@ -504,17 +421,19 @@ jump(i1) """ ops = parse(loop) - self.cpu.assembler.set_debug(True) - self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) - self.cpu.set_future_value_int(0, 0) - self.cpu.execute_token(ops.token) - # check debugging info - name, struct = self.cpu.assembler.loop_run_counters[0] - assert name == 0 # 'xyz' - assert struct.i == 10 - self.cpu.finish_once() - lines = py.path.local(self.logfile + ".count").readlines() - assert lines[0] == '0:10\n' # '10 xyz\n' + debug._log = dlog = debug.DebugLog() + try: + self.cpu.assembler.set_debug(True) + self.cpu.compile_loop(ops.inputargs, ops.operations, ops.token) + self.cpu.set_future_value_int(0, 0) + self.cpu.execute_token(ops.token) + # check debugging info + struct = self.cpu.assembler.loop_run_counters[0] + assert struct.i == 10 + self.cpu.finish_once() + finally: + debug._log = None + assert ('jit-backend-counts', [('debug_print', '0:10')]) in dlog def test_debugger_checksum(self): loop = """ Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_rx86.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_rx86.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/x86/test/test_rx86.py Sat Dec 11 15:10:15 2010 @@ -5,7 +5,6 @@ class CodeBuilderMixin(object): def __init__(self): self.buffer = [] - self.base_address = 0x76543210 def writechar(self, c): assert isinstance(c, str) and len(c) == 1 @@ -14,9 +13,6 @@ def getvalue(self): return ''.join(self.buffer) - def tell(self): - return self.base_address + len(self.buffer) - def assert_encodes_as(code_builder_cls, insn_name, args, expected_encoding): s = code_builder_cls() getattr(s, insn_name)(*args) @@ -104,21 +100,18 @@ def test_call_l(s=None): s = s or CodeBuilder32() - s.CALL_l(0x01234567) - ofs = 0x01234567 - (0x76543210+5) - assert s.getvalue() == '\xE8' + struct.pack(" 5: + x = A1 + else: + x = A2 + x() + rtyper = support.annotate(f, [35]) + maingraph = rtyper.annotator.translator.graphs[0] + cw = CodeWriter(FakeCPU(rtyper), [FakeJitDriverSD(maingraph)]) + cw.find_all_graphs(MyFakePolicy()) + cw.make_jitcodes(verbose=True) + # + names = [jitcode.name for jitcode in cw.assembler.indirectcalltargets] + assert len(names) == 1 + assert names[0].startswith('instantiate_') and names[0].endswith('A1') + # + print cw.assembler.list_of_addr2name + names = dict.fromkeys([value + for key, value in cw.assembler.list_of_addr2name]) + assert 'A1' in names + assert 'A2' in names + def test_int_abs(): def f(n): return abs(n) Modified: pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_list.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_list.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_list.py Sat Dec 11 15:10:15 2010 @@ -56,9 +56,8 @@ if '/' in oopspec_name: oopspec_name, property = oopspec_name.split('/') def force_flags(op): - if property == 'NONNEG': return True, False - if property == 'NEG': return False, False - if property == 'CANRAISE': return False, True + if property == 'NONNEG': return True + if property == 'NEG': return False raise ValueError(property) tr._get_list_nonneg_canraise_flags = force_flags op = SpaceOperation('direct_call', @@ -122,9 +121,6 @@ check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_i %r0, , %i1 -> %i2 """) - builtin_test('list.getitem/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -139,9 +135,6 @@ check_neg_index %r0, , %i0 -> %i1 getarrayitem_gc_pure_i %r0, , %i1 -> %i2 """) - builtin_test('list.getitem_foldable/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -158,10 +151,6 @@ check_neg_index %r0, , %i0 -> %i1 setarrayitem_gc_i %r0, , %i1, %i2 """) - builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), - varoftype(lltype.Signed), - varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -206,9 +195,6 @@ check_resizable_neg_index %r0, , %i0 -> %i1 getlistitem_gc_i %r0, , , %i1 -> %i2 """) - builtin_test('list.getitem/CANRAISE', - [varoftype(VARLIST), varoftype(lltype.Signed)], - lltype.Signed, NotSupported) def test_resizable_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), @@ -225,10 +211,6 @@ check_resizable_neg_index %r0, , %i0 -> %i1 setlistitem_gc_i %r0, , , %i1, %i2 """) - builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), - varoftype(lltype.Signed), - varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_resizable_len(): builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, Modified: pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_regalloc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_regalloc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_regalloc.py Sat Dec 11 15:10:15 2010 @@ -281,22 +281,22 @@ # this used to produce bogus code, containing these two # lines in the following broken order: # last_exc_value -> %r0 - # ref_copy %r0 -> %r2 -- but expect to read the old value of %r0! + # ref_copy %r0 -> %r1 -- but expect to read the old value of %r0! self.check_assembler(graph, """ residual_call_r_r $<* fn bar>, , R[%r0] -> %r1 -live- - residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r2 + residual_call_ir_r $<* fn g>, , I[%i0], R[] -> %r1 -live- catch_exception L1 - ref_return %r2 + ref_return %r1 --- L1: goto_if_exception_mismatch $<* struct object_vtable>, L2 - ref_copy %r0 -> %r2 + ref_copy %r0 -> %r1 last_exc_value -> %r0 residual_call_r_r $<* fn foo>, , R[%r0] -> %r0 -live- - ref_return %r2 + ref_return %r1 --- L2: reraise Modified: pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_void_list.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_void_list.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/codewriter/test/test_void_list.py Sat Dec 11 15:10:15 2010 @@ -51,9 +51,6 @@ builtin_test('list.getitem/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_getitem_foldable(): builtin_test('list.getitem_foldable/NONNEG', @@ -62,9 +59,6 @@ builtin_test('list.getitem_foldable/NEG', [varoftype(FIXEDLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem_foldable/CANRAISE', - [varoftype(FIXEDLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_fixed_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(FIXEDLIST), @@ -75,10 +69,6 @@ varoftype(lltype.Signed), varoftype(lltype.Void)], lltype.Void, "") - builtin_test('list.setitem/CANRAISE', [varoftype(FIXEDLIST), - varoftype(lltype.Signed), - varoftype(lltype.Void)], - lltype.Void, NotSupported) def test_fixed_len(): builtin_test('list.len', [varoftype(FIXEDLIST)], lltype.Signed, @@ -115,9 +105,6 @@ builtin_test('list.getitem/NEG', [varoftype(VARLIST), varoftype(lltype.Signed)], lltype.Void, "") - builtin_test('list.getitem/CANRAISE', - [varoftype(VARLIST), varoftype(lltype.Signed)], - lltype.Void, NotSupported) def test_resizable_setitem(): builtin_test('list.setitem/NONNEG', [varoftype(VARLIST), @@ -128,10 +115,6 @@ varoftype(lltype.Signed), varoftype(lltype.Void)], lltype.Void, "") - builtin_test('list.setitem/CANRAISE', [varoftype(VARLIST), - varoftype(lltype.Signed), - varoftype(lltype.Void)], - lltype.Void, NotSupported) def test_resizable_len(): builtin_test('list.len', [varoftype(VARLIST)], lltype.Signed, Modified: pypy/branch/jit-unroll-loops/pypy/jit/conftest.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/conftest.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/conftest.py Sat Dec 11 15:10:15 2010 @@ -1,7 +1,5 @@ import py -option = py.test.config.option - def pytest_addoption(parser): group = parser.getgroup("JIT options") group.addoption('--slow', action="store_true", Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py Sat Dec 11 15:10:15 2010 @@ -1,4 +1,4 @@ - +import weakref from pypy.rpython.lltypesystem import lltype from pypy.rpython.ootypesystem import ootype from pypy.objspace.flow.model import Constant, Variable @@ -46,6 +46,32 @@ loop_token.outermost_jitdriver_sd = jitdriver_sd return loop_token +def record_loop_or_bridge(loop): + """Do post-backend recordings and cleanups on 'loop'. + """ + # get the original loop token (corresponding to 'loop', or if that is + # a bridge, to the loop that this bridge belongs to) + looptoken = loop.token + assert looptoken is not None + wref = weakref.ref(looptoken) + for op in loop.operations: + descr = op.getdescr() + if isinstance(descr, ResumeDescr): + descr.wref_original_loop_token = wref # stick it there + n = descr.index + if n >= 0: # we also record the resumedescr number + looptoken.compiled_loop_token.record_faildescr_index(n) + elif isinstance(descr, LoopToken): + # for a JUMP or a CALL_ASSEMBLER: record it as a potential jump. + # (the following test is not enough to prevent more complicated + # cases of cycles, but at least it helps in simple tests of + # test_memgr.py) + if descr is not looptoken: + looptoken.record_jump_to(descr) + op.setdescr(None) # clear reference, mostly for tests + # mostly for tests: make sure we don't keep a reference to the LoopToken + loop.token = None + # ____________________________________________________________ def compile_new_loop(metainterp, old_loop_tokens, greenkey, start, @@ -82,15 +108,18 @@ if loop.preamble.operations is not None: send_loop_to_backend(metainterp_sd, loop, "loop") + record_loop_or_bridge(loop) if full_preamble_needed or not loop.preamble.token.short_preamble: send_loop_to_backend(metainterp_sd, loop.preamble, "entry bridge") insert_loop_token(old_loop_tokens, loop.preamble.token) + record_loop_or_bridge(loop.preamble) jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( greenkey, loop.preamble.token) return loop.preamble.token else: send_loop_to_backend(metainterp_sd, loop, "loop") insert_loop_token(old_loop_tokens, loop_token) + record_loop_or_bridge(loop) jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( greenkey, loop.token) return loop_token @@ -133,8 +162,11 @@ else: loop._ignore_during_counting = True metainterp_sd.log("compiled new " + type) + if metainterp_sd.warmrunnerdesc is not None: # for tests + metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(loop.token) -def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations): +def send_bridge_to_backend(metainterp_sd, faildescr, inputargs, operations, + original_loop_token): n = metainterp_sd.cpu.get_fail_descr_number(faildescr) metainterp_sd.logger_ops.log_bridge(inputargs, operations, n) if not we_are_translated(): @@ -143,7 +175,8 @@ metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: - metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations) + metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, + original_loop_token) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() @@ -240,9 +273,6 @@ CNT_FLOAT = -0x60000000 CNT_MASK = 0x1FFFFFFF - def __init__(self, metainterp_sd): - self.metainterp_sd = metainterp_sd - def store_final_boxes(self, guard_op, boxes): guard_op.setfailargs(boxes) self.guard_opnum = guard_op.getopnum() @@ -327,12 +357,14 @@ def compile_and_attach(self, metainterp, new_loop): # We managed to create a bridge. Attach the new operations - # to the corrsponding guard_op and compile from there + # to the corresponding guard_op and compile from there + assert metainterp.resumekey_original_loop_token is not None + new_loop.token = metainterp.resumekey_original_loop_token inputargs = metainterp.history.inputargs if not we_are_translated(): self._debug_suboperations = new_loop.operations send_bridge_to_backend(metainterp.staticdata, self, inputargs, - new_loop.operations) + new_loop.operations, new_loop.token) def copy_all_attrbutes_into(self, res): # XXX a bit ugly to have to list them all here @@ -344,14 +376,14 @@ res.rd_pendingfields = self.rd_pendingfields def _clone_if_mutable(self): - res = ResumeGuardDescr(self.metainterp_sd) + res = ResumeGuardDescr() self.copy_all_attrbutes_into(res) return res class ResumeGuardForcedDescr(ResumeGuardDescr): def __init__(self, metainterp_sd, jitdriver_sd): - ResumeGuardDescr.__init__(self, metainterp_sd) + self.metainterp_sd = metainterp_sd self.jitdriver_sd = jitdriver_sd def handle_fail(self, metainterp_sd, jitdriver_sd): @@ -495,6 +527,8 @@ metainterp_sd = metainterp.staticdata jitdriver_sd = metainterp.jitdriver_sd redargs = new_loop.inputargs + # We make a new LoopToken for this entry bridge, and stick it + # to every guard in the loop. new_loop_token = make_loop_token(len(redargs), jitdriver_sd) new_loop.token = new_loop_token send_loop_to_backend(metainterp_sd, new_loop, "entry bridge") @@ -502,12 +536,14 @@ jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( self.original_greenkey, new_loop_token) - # store the new loop in compiled_merge_points too + # store the new loop in compiled_merge_points_wref too old_loop_tokens = metainterp.get_compiled_merge_points( self.original_greenkey) # it always goes at the end of the list, as it is the most # general loop token old_loop_tokens.append(new_loop_token) + metainterp.set_compiled_merge_points(self.original_greenkey, + old_loop_tokens) def reset_counter_from_failure(self): pass @@ -543,6 +579,7 @@ prepare_last_operation(new_loop, target_loop_token) resumekey.compile_and_attach(metainterp, new_loop) compile_known_target_bridges(metainterp, new_loop) + record_loop_or_bridge(new_loop) return target_loop_token # For backends that not supports emitting guards with preset jump @@ -564,6 +601,7 @@ descr._debug_suboperations = mini.operations send_bridge_to_backend(metainterp.staticdata, descr, mini.inputargs, mini.operations) + record_loop_or_bridge(mini) def prepare_last_operation(new_loop, target_loop_token): @@ -591,7 +629,8 @@ propagate_exception_descr = PropagateExceptionDescr() -def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redboxes): +def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redboxes, + memory_manager=None): """Make a LoopToken that corresponds to assembler code that just calls back the interpreter. Used temporarily: a fully compiled version of the code may end up replacing it. @@ -631,4 +670,6 @@ ] operations[1].setfailargs([]) cpu.compile_loop(inputargs, operations, loop_token, log=False) + if memory_manager is not None: # for tests + memory_manager.keep_loop_alive(loop_token) return loop_token Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/history.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/history.py Sat Dec 11 15:10:15 2010 @@ -4,8 +4,7 @@ from pypy.rpython.ootypesystem import ootype from pypy.rlib.objectmodel import we_are_translated, r_dict, Symbolic from pypy.rlib.objectmodel import compute_hash, compute_unique_id -from pypy.rlib.rarithmetic import intmask -from pypy.tool.uid import uid +from pypy.rlib.rarithmetic import intmask, r_int64 from pypy.conftest import option from pypy.jit.metainterp.resoperation import ResOperation, rop @@ -745,14 +744,28 @@ terminating = False # see TerminatingLoopToken in compile.py outermost_jitdriver_sd = None # and more data specified by the backend when the loop is compiled - number = 0 + number = -1 + generation = r_int64(0) + # one purpose of LoopToken is to keep alive the CompiledLoopToken + # returned by the backend. When the LoopToken goes away, the + # CompiledLoopToken has its __del__ called, which frees the assembler + # memory and the ResumeGuards. + compiled_loop_token = None - def __init__(self, number=0): - self.number = number + def __init__(self): + # For memory management of assembled loops + self._keepalive_target_looktokens = {} # set of other LoopTokens + + def record_jump_to(self, target_loop_token): + self._keepalive_target_looktokens[target_loop_token] = None + + def __repr__(self): + return '' % (self.number, self.generation) def repr_of_descr(self): return '' % self.number + class TreeLoop(object): inputargs = None operations = None Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/jitprof.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/jitprof.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/jitprof.py Sat Dec 11 15:10:15 2010 @@ -2,9 +2,9 @@ """ A small helper module for profiling JIT """ -import os import time -from pypy.rlib.debug import debug_print +from pypy.rlib.debug import debug_print, debug_start, debug_stop +from pypy.rlib.debug import have_debug_prints from pypy.jit.metainterp.jitexc import JitException counters=""" @@ -24,6 +24,10 @@ NVIRTUALS NVHOLES NVREUSED +TOTAL_COMPILED_LOOPS +TOTAL_COMPILED_BRIDGES +TOTAL_FREED_LOOPS +TOTAL_FREED_BRIDGES """ def _setup(): @@ -35,6 +39,7 @@ _setup() JITPROF_LINES = ncounters + 1 + 1 # one for TOTAL, 1 for calls, update if needed +_CPU_LINES = 4 # the last 4 lines are stored on the cpu class BaseProfiler(object): pass @@ -48,9 +53,6 @@ def finish(self): pass - def set_printing(self, printing): - pass - def start_tracing(self): pass @@ -90,23 +92,19 @@ counters = None calls = 0 current = None - printing = True + cpu = None def start(self): self.starttime = self.timer() self.t1 = self.starttime self.times = [0, 0] - self.counters = [0] * ncounters + self.counters = [0] * (ncounters - _CPU_LINES) self.calls = 0 self.current = [] def finish(self): self.tk = self.timer() - if self.printing: - self.print_stats() - - def set_printing(self, printing): - self.printing = printing + self.print_stats() def _start(self, event): t0 = self.t1 @@ -154,6 +152,12 @@ self.calls += 1 def print_stats(self): + debug_start("jit-summary") + if have_debug_prints(): + self._print_stats() + debug_stop("jit-summary") + + def _print_stats(self): cnt = self.counters tim = self.times calls = self.calls @@ -161,8 +165,8 @@ self._print_line_time("Backend", cnt[BACKEND], tim[BACKEND]) self._print_intline("Running asm", cnt[RUNNING]) self._print_intline("Blackhole", cnt[BLACKHOLE]) - line = "TOTAL: \t\t%f\n" % (self.tk - self.starttime, ) - os.write(2, line) + line = "TOTAL: \t\t%f" % (self.tk - self.starttime, ) + debug_print(line) self._print_intline("ops", cnt[OPS]) self._print_intline("recorded ops", cnt[RECORDED_OPS]) self._print_intline(" calls", calls) @@ -176,17 +180,26 @@ self._print_intline("nvirtuals", cnt[NVIRTUALS]) self._print_intline("nvholes", cnt[NVHOLES]) self._print_intline("nvreused", cnt[NVREUSED]) + cpu = self.cpu + if cpu is not None: # for some tests + self._print_intline("Total # of loops", + cpu.total_compiled_loops) + self._print_intline("Total # of bridges", + cpu.total_compiled_bridges) + self._print_intline("Freed # of loops", + cpu.total_freed_loops) + self._print_intline("Freed # of bridges", + cpu.total_freed_bridges) def _print_line_time(self, string, i, tim): - final = "%s:%s\t%d\t%f\n" % (string, " " * max(0, 13-len(string)), i, tim) - os.write(2, final) + final = "%s:%s\t%d\t%f" % (string, " " * max(0, 13-len(string)), i, tim) + debug_print(final) def _print_intline(self, string, i): final = string + ':' + " " * max(0, 16-len(string)) - final += '\t' + str(i) + '\n' - os.write(2, final) - - + final += '\t' + str(i) + debug_print(final) + class BrokenProfilerData(JitException): pass Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/__init__.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/__init__.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/__init__.py Sat Dec 11 15:10:15 2010 @@ -3,7 +3,6 @@ from pypy.jit.metainterp.optimizeopt.intbounds import OptIntBounds from pypy.jit.metainterp.optimizeopt.virtualize import OptVirtualize from pypy.jit.metainterp.optimizeopt.heap import OptHeap -from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall from pypy.jit.metainterp.optimizeopt.string import OptString from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble @@ -17,6 +16,10 @@ OptVirtualize(), opt_str, OptHeap(), + ] + if metainterp_sd.jit_ffi: + from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall + optimizations = optimizations + [ OptFfiCall(), ] Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/optimizer.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/optimizer.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/optimizer.py Sat Dec 11 15:10:15 2010 @@ -379,6 +379,10 @@ return CVAL_ZERO def propagate_all_forward(self): + self.exception_might_have_happened = True + # ^^^ at least at the start of bridges. For loops, we could set + # it to False, but we probably don't care + self.newoperations = [] self.i = 0 while self.i < len(self.loop.operations): op = self.loop.operations[self.i] Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/pyjitpl.py Sat Dec 11 15:10:15 2010 @@ -14,11 +14,10 @@ from pypy.jit.metainterp.logger import Logger from pypy.jit.metainterp.jitprof import EmptyProfiler from pypy.jit.metainterp.jitprof import GUARDS, RECORDED_OPS, ABORT_ESCAPE -from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG +from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE from pypy.jit.metainterp.jitexc import JitException, get_llexception from pypy.rlib.rarithmetic import intmask from pypy.rlib.objectmodel import specialize -from pypy.rlib.jit import DEBUG_OFF, DEBUG_PROFILE, DEBUG_STEPS, DEBUG_DETAILED from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr from pypy.jit.codewriter import heaptracker from pypy.jit.metainterp.optimizeutil import RetraceLoop @@ -95,6 +94,18 @@ else: raise AssertionError(argcode) outvalue[startindex+i] = reg + def _put_back_list_of_boxes(self, outvalue, startindex, position): + code = self.bytecode + length = ord(code[position]) + position += 1 + for i in range(length): + index = ord(code[position+i]) + box = outvalue[startindex+i] + if box.type == history.INT: self.registers_i[index] = box + elif box.type == history.REF: self.registers_r[index] = box + elif box.type == history.FLOAT: self.registers_f[index] = box + else: raise AssertionError(box.type) + def get_current_position_info(self): return self.jitcode.get_live_vars_info(self.pc) @@ -602,8 +613,10 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) arrayindex = vinfo.array_field_by_descrs[arrayfielddescr] index = indexbox.getint() - if index < 0: - index += vinfo.get_array_length(virtualizable, arrayindex) + # Support for negative index: disabled + # (see codewriter/jtransform.py, _check_no_vable_array). + #if index < 0: + # index += vinfo.get_array_length(virtualizable, arrayindex) assert 0 <= index < vinfo.get_array_length(virtualizable, arrayindex) return vinfo.get_index_in_array(virtualizable, arrayindex, index) @@ -815,8 +828,9 @@ for i in range(num_green_args): assert isinstance(varargs[i], Const) - @arguments("orgpc", "int", "boxes3", "boxes3") - def opimpl_jit_merge_point(self, orgpc, jdindex, greenboxes, redboxes): + @arguments("orgpc", "int", "boxes3", "jitcode_position", "boxes3") + def opimpl_jit_merge_point(self, orgpc, jdindex, greenboxes, + jcposition, redboxes): any_operation = len(self.metainterp.history.operations) > 0 jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex] self.verify_green_args(jitdriver_sd, greenboxes) @@ -844,6 +858,10 @@ self.pc = orgpc self.metainterp.reached_loop_header(greenboxes, redboxes) self.pc = saved_pc + # no exception, which means that the jit_merge_point did not + # close the loop. We have to put the possibly-modified list + # 'redboxes' back into the registers where it comes from. + put_back_list_of_boxes3(self, jcposition, redboxes) else: # warning! careful here. We have to return from the current # frame containing the jit_merge_point, and then use @@ -1053,7 +1071,7 @@ resumedescr = compile.ResumeGuardForcedDescr(metainterp_sd, metainterp.jitdriver_sd) else: - resumedescr = compile.ResumeGuardDescr(metainterp_sd) + resumedescr = compile.ResumeGuardDescr() guard_op = metainterp.history.record(opnum, moreargs, None, descr=resumedescr) virtualizable_boxes = None @@ -1199,7 +1217,8 @@ logger_ops = None def __init__(self, cpu, options, - ProfilerClass=EmptyProfiler, warmrunnerdesc=None): + ProfilerClass=EmptyProfiler, warmrunnerdesc=None, + jit_ffi=True): self.cpu = cpu self.stats = self.cpu.stats self.options = options @@ -1207,7 +1226,9 @@ self.logger_ops = Logger(self, guard_number=True) self.profiler = ProfilerClass() + self.profiler.cpu = cpu self.warmrunnerdesc = warmrunnerdesc + self.jit_ffi = jit_ffi backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] @@ -1325,6 +1346,11 @@ return jitcode return None + def try_to_free_some_loops(self): + # Increase here the generation recorded by the memory manager. + if self.warmrunnerdesc is not None: # for tests + self.warmrunnerdesc.memory_manager.next_generation() + # ---------------- logging ------------------------ def log(self, msg): @@ -1627,6 +1653,7 @@ self.staticdata._setup_once() self.staticdata.profiler.start_tracing() assert jitdriver_sd is self.jitdriver_sd + self.staticdata.try_to_free_some_loops() self.create_empty_history() try: original_boxes = self.initialize_original_boxes(jitdriver_sd, *args) @@ -1655,10 +1682,15 @@ debug_start('jit-tracing') self.staticdata.profiler.start_tracing() assert isinstance(key, compile.ResumeGuardDescr) + # store the resumekey.wref_original_loop_token() on 'self' to make + # sure that it stays alive as long as this MetaInterp + self.resumekey_original_loop_token = key.wref_original_loop_token() + self.staticdata.try_to_free_some_loops() self.initialize_state_from_guard_failure(key) try: return self._handle_guard_failure(key) finally: + self.resumekey_original_loop_token = None self.staticdata.profiler.end_tracing() debug_stop('jit-tracing') @@ -1668,6 +1700,8 @@ self.seen_loop_header_for_jdindex = -1 try: self.prepare_resume_from_failure(key.guard_opnum) + if self.resumekey_original_loop_token is None: # very rare case + raise SwitchToBlackhole(ABORT_BRIDGE) self.interpret() except GenerateMergePoint, gmp: return self.designate_target_loop(gmp) @@ -1803,10 +1837,15 @@ raise NotImplementedError(opname[opnum]) def get_compiled_merge_points(self, greenkey): + """Get the list of looptokens corresponding to the greenkey. + Turns the (internal) list of weakrefs into regular refs. + """ cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) - if cell.compiled_merge_points is None: - cell.compiled_merge_points = [] - return cell.compiled_merge_points + return cell.get_compiled_merge_points() + + def set_compiled_merge_points(self, greenkey, looptokens): + cell = self.jitdriver_sd.warmstate.jit_cell_at_key(greenkey) + cell.set_compiled_merge_points(looptokens) def compile(self, original_boxes, live_arg_boxes, start): num_green_args = self.jitdriver_sd.num_green_args @@ -1817,6 +1856,7 @@ loop_token = compile.compile_new_loop(self, old_loop_tokens, greenkey, start) if loop_token is not None: # raise if it *worked* correctly + self.set_compiled_merge_points(greenkey, old_loop_tokens) raise GenerateMergePoint(live_arg_boxes, loop_token) self.history.operations.pop() # remove the JUMP # FIXME: Why is self.history.inputargs not restored? @@ -2345,6 +2385,8 @@ else: raise AssertionError("bad argcode") position += 1 + elif argtype == "jitcode_position": + value = position else: raise AssertionError("bad argtype: %r" % (argtype,)) args += (value,) @@ -2389,3 +2431,15 @@ argtypes = unrolling_iterable(unboundmethod.argtypes) handler.func_name = 'handler_' + name return handler + +def put_back_list_of_boxes3(frame, position, newvalue): + code = frame.bytecode + length1 = ord(code[position]) + position2 = position + 1 + length1 + length2 = ord(code[position2]) + position3 = position2 + 1 + length2 + length3 = ord(code[position3]) + assert len(newvalue) == length1 + length2 + length3 + frame._put_back_list_of_boxes(newvalue, 0, position) + frame._put_back_list_of_boxes(newvalue, length1, position2) + frame._put_back_list_of_boxes(newvalue, length1 + length2, position3) Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_basic.py Sat Dec 11 15:10:15 2010 @@ -19,7 +19,11 @@ from pypy.jit.metainterp import simple_optimize class FakeJitCell: - compiled_merge_points = None + __compiled_merge_points = [] + def get_compiled_merge_points(self): + return self.__compiled_merge_points[:] + def set_compiled_merge_points(self, lst): + self.__compiled_merge_points = lst class FakeWarmRunnerState: def attach_unoptimized_bridge_from_interp(self, greenkey, newloop): @@ -35,7 +39,6 @@ optimize_bridge = staticmethod(simple_optimize.optimize_bridge) trace_limit = sys.maxint - debug_level = 2 func._jit_unroll_safe_ = True rtyper = support.annotate(func, values, type_system=type_system) @@ -1996,6 +1999,29 @@ res = self.meta_interp(f, [5, 2]) assert 4 < res < 14 + def test_compute_identity_hash(self): + from pypy.rlib.objectmodel import compute_identity_hash + class A(object): + pass + def f(): + a = A() + return compute_identity_hash(a) == compute_identity_hash(a) + res = self.interp_operations(f, []) + assert res + # a "did not crash" kind of test + + def test_compute_unique_id(self): + from pypy.rlib.objectmodel import compute_unique_id + class A(object): + pass + def f(): + a1 = A() + a2 = A() + return (compute_unique_id(a1) == compute_unique_id(a1) and + compute_unique_id(a1) != compute_unique_id(a2)) + res = self.interp_operations(f, []) + assert res + class TestOOtype(BasicTests, OOJitMixin): Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_compile.py Sat Dec 11 15:10:15 2010 @@ -39,7 +39,6 @@ class FakeState: optimize_loop = staticmethod(nounroll_optimize.optimize_loop) - debug_level = 0 def attach_unoptimized_bridge_from_interp(*args): pass @@ -54,6 +53,8 @@ stats = Stats() profiler = jitprof.EmptyProfiler() + warmrunnerdesc = None + jit_ffi = False def log(self, msg, event_kind=None): pass @@ -207,14 +208,12 @@ class ExitFrameWithExceptionRef(Exception): pass FakeMetaInterpSD.cpu = cpu - class FakeJitDriverSD: - pass cpu.set_future_value_int(0, -156) cpu.set_future_value_int(1, -178) cpu.set_future_value_int(2, -190) fail_descr = cpu.execute_token(loop_token) try: - fail_descr.handle_fail(FakeMetaInterpSD(), FakeJitDriverSD()) + fail_descr.handle_fail(FakeMetaInterpSD(), None) except FakeMetaInterpSD.ExitFrameWithExceptionRef, e: assert lltype.cast_opaque_ptr(lltype.Ptr(EXC), e.args[1]) == llexc else: Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_exception.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_exception.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_exception.py Sat Dec 11 15:10:15 2010 @@ -1,6 +1,6 @@ import py, sys from pypy.jit.metainterp.test.test_basic import LLJitMixin, OOJitMixin -from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE +from pypy.rlib.jit import JitDriver, OPTIMIZER_SIMPLE, dont_look_inside from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask from pypy.jit.codewriter.policy import StopAtXPolicy @@ -587,6 +587,33 @@ res = self.interp_operations(f, [99]) assert res == 21 + def test_bug_exc1_noexc_exc2(self): + myjitdriver = JitDriver(greens=[], reds=['i']) + @dont_look_inside + def rescall(i): + if i < 10: + raise KeyError + if i < 20: + return None + raise ValueError + def f(i): + while i < 30: + myjitdriver.can_enter_jit(i=i) + myjitdriver.jit_merge_point(i=i) + try: + rescall(i) + except KeyError: + assert i < 10 + except ValueError: + assert i >= 20 + else: + assert 10 <= i < 20 + i += 1 + return i + res = self.meta_interp(f, [0], inline=True) + assert res == 30 + + class MyError(Exception): def __init__(self, n): self.n = n Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_fficall.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_fficall.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_fficall.py Sat Dec 11 15:10:15 2010 @@ -40,6 +40,6 @@ n += 1 return res # - res = self.meta_interp(f, [0]) + res = self.meta_interp(f, [0], jit_ffi=True) return res Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_list.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_list.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_list.py Sat Dec 11 15:10:15 2010 @@ -188,6 +188,26 @@ assert res == f(4) self.check_loops(call=0, getfield_gc=0) + def test_fold_indexerror(self): + jitdriver = JitDriver(greens = [], reds = ['total', 'n', 'lst']) + def f(n): + lst = [] + total = 0 + while n > 0: + jitdriver.can_enter_jit(lst=lst, n=n, total=total) + jitdriver.jit_merge_point(lst=lst, n=n, total=total) + lst.append(n) + try: + total += lst[n] + except IndexError: + total += 1000 + n -= 1 + return total + + res = self.meta_interp(f, [15], listops=True) + assert res == f(15) + self.check_loops(guard_exception=0) + class TestOOtype(ListTests, OOJitMixin): pass Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_logger.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_logger.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_logger.py Sat Dec 11 15:10:15 2010 @@ -14,11 +14,20 @@ def capturing(func, *args, **kwds): log_stream = StringIO() - debug._stderr = log_stream + class MyDebugLog: + def debug_print(self, *args): + for arg in args: + print >> log_stream, arg, + print >> log_stream + def debug_start(self, *args): + pass + def debug_stop(self, *args): + pass try: + debug._log = MyDebugLog() func(*args, **kwds) finally: - debug._stderr = sys.stderr + debug._log = None return log_stream.getvalue() class Logger(logger.Logger): @@ -112,7 +121,8 @@ equaloplists(loop.operations, oloop.operations) def test_jump(self): - namespace = {'target': LoopToken(3)} + namespace = {'target': LoopToken()} + namespace['target'].number = 3 inp = ''' [i0] jump(i0, descr=target) Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizefficall.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizefficall.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizefficall.py Sat Dec 11 15:10:15 2010 @@ -32,6 +32,7 @@ class TestFfiCall(BaseTestBasic, LLtypeMixin): + jit_ffi = True class namespace: cpu = LLtypeMixin.cpu Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py Sat Dec 11 15:10:15 2010 @@ -21,11 +21,12 @@ class FakeMetaInterpStaticData(object): - def __init__(self, cpu): + def __init__(self, cpu, jit_ffi=False): self.cpu = cpu self.profiler = EmptyProfiler() self.options = Fake() self.globaldata = Fake() + self.jit_ffi = jit_ffi def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr @@ -34,7 +35,7 @@ b1 = BoxInt() opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu), None) - fdescr = ResumeGuardDescr(None) + fdescr = ResumeGuardDescr() op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr) # setup rd data fi0 = resume.FrameInfo(None, "code0", 11) @@ -141,6 +142,7 @@ return sorted(boxes, key=lambda box: _kind2count[box.type]) class BaseTestOptimizeOpt(BaseTest): + jit_ffi = False def invent_fail_descr(self, fail_args): if fail_args is None: @@ -169,7 +171,7 @@ loop.preamble = TreeLoop('preamble') loop.preamble.inputargs = loop.inputargs loop.preamble.token = LoopToken() - metainterp_sd = FakeMetaInterpStaticData(self.cpu) + metainterp_sd = FakeMetaInterpStaticData(self.cpu, self.jit_ffi) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo if hasattr(self, 'callinfocollection'): @@ -882,8 +884,12 @@ i3 = call(i2, descr=nonwritedescr) jump(i1) # the exception is considered lost when we loop back """ + # note that 'guard_no_exception' at the very start must be kept + # around: bridges may start with one. (In case of loops we could + # remove it, but we probably don't care.) expected = """ [i] + guard_no_exception() [] i1 = int_add(i, 3) i2 = call(i1, descr=nonwritedescr) guard_no_exception() [i1, i2] @@ -2227,6 +2233,15 @@ """ self.optimize_loop(ops, expected, preamble) + def test_bug_4(self): + ops = """ + [p9] + p30 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) + jump(p30) + """ + self.optimize_loop(ops, 'Not', ops) + def test_invalid_loop_1(self): ops = """ [p1] Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_pyjitpl.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_pyjitpl.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_pyjitpl.py Sat Dec 11 15:10:15 2010 @@ -17,6 +17,7 @@ portal.setup(None) class FakeStaticData: cpu = None + warmrunnerdesc = None metainterp = pyjitpl.MetaInterp(FakeStaticData(), None) metainterp.framestack = [] @@ -53,6 +54,7 @@ def test_remove_consts_and_duplicates(): class FakeStaticData: cpu = None + warmrunnerdesc = None def is_another_box_like(box, referencebox): assert box is not referencebox assert isinstance(box, referencebox.clonebox().__class__) Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_recursive.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_recursive.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_recursive.py Sat Dec 11 15:10:15 2010 @@ -927,12 +927,16 @@ x=x) frame.s = hint(frame.s, promote=True) n -= 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] frame.s += 1 if codeno == 0: subframe = Frame([n, n+1, n+2, n+3], 0) x += f(1, 10, 1, subframe) - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] x += len(frame.l) frame.s -= 1 return x @@ -1142,6 +1146,19 @@ res = self.meta_interp(main, [], inline=True, trace_limit=tlimit) assert ''.join(res.chars) == 'ABCDEFGHIabcdefghijJ' * 5 + def test_no_duplicates_bug(self): + driver = JitDriver(greens = ['codeno'], reds = ['i'], + get_printable_location = lambda codeno: str(codeno)) + def portal(codeno, i): + while i > 0: + driver.can_enter_jit(codeno=codeno, i=i) + driver.jit_merge_point(codeno=codeno, i=i) + if codeno > 0: + break + portal(i, i) + i -= 1 + self.meta_interp(portal, [0, 10], inline=True) + class TestLLtype(RecursiveTests, LLJitMixin): pass Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_virtualizable.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_virtualizable.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_virtualizable.py Sat Dec 11 15:10:15 2010 @@ -481,9 +481,13 @@ myjitdriver.jit_merge_point(frame=frame, n=n, x=x) frame.s = hint(frame.s, promote=True) n -= 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] frame.s += 1 - x += frame.l[frame.s] + s = frame.s + assert s >= 0 + x += frame.l[s] x += len(frame.l) frame.s -= 1 return x @@ -995,7 +999,9 @@ jitdriver.can_enter_jit(frame=frame, n=n) jitdriver.jit_merge_point(frame=frame, n=n) popped = frame.stack[frame.stackpos] - frame.stackpos -= 1 + sp = frame.stackpos - 1 + assert sp >= 0 + frame.stackpos = sp to_push = intmask(popped * 3) frame.stack[frame.stackpos] = to_push frame.stackpos += 1 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_warmspot.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_warmspot.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_warmspot.py Sat Dec 11 15:10:15 2010 @@ -132,91 +132,6 @@ assert state.optimize_loop is optimize.optimize_loop assert state.optimize_bridge is optimize.optimize_bridge - def test_static_debug_level(self, capfd): - py.test.skip("debug_level is being deprecated") - from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS - from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler - - myjitdriver = JitDriver(greens = [], reds = ['n']) - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - return n - - capfd.readouterr() - self.meta_interp(f, [10], debug_level=DEBUG_OFF, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not "Running asm" in err - self.meta_interp(f, [10], debug_level=DEBUG_PROFILE, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not 'compiled new' in err - assert "Running asm" in err - - self.meta_interp(f, [10], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, err = capfd.readouterr() - assert 'ENTER' in err - assert 'LEAVE' in err - assert "Running asm" in err - - self.meta_interp(f, [10], debug_level=DEBUG_STEPS, - ProfilerClass=EmptyProfiler) - out, err = capfd.readouterr() - assert 'ENTER' in err - assert 'LEAVE' in err - assert not "Running asm" in err - - def test_set_param_debug(self): - py.test.skip("debug_level is being deprecated") - from pypy.rlib.jit import DEBUG_PROFILE, DEBUG_OFF, DEBUG_STEPS - from pypy.jit.metainterp.jitprof import EmptyProfiler, Profiler - - myjitdriver = JitDriver(greens = [], reds = ['n']) - def f(n): - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - return n - - def main(n, debug): - myjitdriver.set_param("debug", debug) - print f(n) - - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_OFF], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not "Running asm" in err - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_PROFILE], debug_level=DEBUG_STEPS, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert not 'ENTER' in err - assert not 'LEAVE' in err - assert not 'compiled new' in err - assert "Running asm" in err - outerr = py.io.StdCaptureFD() - self.meta_interp(main, [10, DEBUG_STEPS], debug_level=DEBUG_OFF, - ProfilerClass=Profiler) - out, errf = outerr.done() - err = errf.read() - assert 'ENTER' in err - assert 'LEAVE' in err - assert "Running asm" in err - def test_unwanted_loops(self): mydriver = JitDriver(reds = ['n', 'total', 'm'], greens = []) @@ -378,23 +293,30 @@ exc_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) cls.exc_vtable = exc_vtable - class FakeFailDescr(object): + class FakeLoopToken: def __init__(self, no): self.no = no + self.generation = 0 + + class FakeFailDescr(object): + def __init__(self, looptoken): + assert isinstance(looptoken, FakeLoopToken) + self.looptoken = looptoken def handle_fail(self, metainterp_sd, jitdrivers_sd): - if self.no == 0: + no = self.looptoken.no + if no == 0: raise metainterp_sd.warmrunnerdesc.DoneWithThisFrameInt(3) - if self.no == 1: + if no == 1: raise metainterp_sd.warmrunnerdesc.ContinueRunningNormally( [0], [], [], [1], [], []) - if self.no == 3: + if no == 3: exc = lltype.malloc(OBJECT) exc.typeptr = exc_vtable raise metainterp_sd.warmrunnerdesc.ExitFrameWithExceptionRef( metainterp_sd.cpu, lltype.cast_opaque_ptr(llmemory.GCREF, exc)) - return self.no + return self.looptoken class FakeDescr: def as_vtable_size_descr(self): @@ -419,11 +341,11 @@ sizeof = nodescr def get_fail_descr_from_number(self, no): - return FakeFailDescr(no) + return FakeFailDescr(FakeLoopToken(no)) def execute_token(self, token): - assert token == 2 - return FakeFailDescr(1) + assert token.no == 2 + return FakeFailDescr(FakeLoopToken(1)) driver = JitDriver(reds = ['red'], greens = ['green']) Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_warmstate.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_warmstate.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_warmstate.py Sat Dec 11 15:10:15 2010 @@ -99,6 +99,8 @@ lltype.Float], lltype.Void)) class FakeWarmRunnerDesc: rtyper = FakeRTyper() + cpu = None + memory_manager = None class FakeJitDriverSD: _get_jitcell_at_ptr = llhelper(GETTER, getter) _set_jitcell_at_ptr = llhelper(SETTER, setter) @@ -126,6 +128,7 @@ future_values[j] = "float", value class FakeWarmRunnerDesc: cpu = FakeCPU() + memory_manager = None class FakeJitDriverSD: _red_args_types = ["int", "float"] virtualizable_info = None @@ -154,16 +157,20 @@ _get_jitcell_at_ptr = None state = WarmEnterState(None, FakeJitDriverSD()) get_jitcell = state.make_jitcell_getter() + class FakeLoopToken(object): + pass + looptoken = FakeLoopToken() state.attach_unoptimized_bridge_from_interp([ConstInt(5), ConstFloat(2.25)], - "entry loop token") + looptoken) cell1 = get_jitcell(True, 5, 2.25) assert cell1.counter < 0 - assert cell1.entry_loop_token == "entry loop token" + assert cell1.get_entry_loop_token() is looptoken def test_make_jitdriver_callbacks_1(): class FakeWarmRunnerDesc: cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None @@ -189,6 +196,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = llhelper(GET_LOCATION, get_location) @@ -211,6 +219,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None @@ -233,6 +242,7 @@ class FakeWarmRunnerDesc: rtyper = None cpu = None + memory_manager = None class FakeJitDriverSD: _green_args_spec = [lltype.Signed, lltype.Float] _get_printable_location_ptr = None Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmspot.py Sat Dec 11 15:10:15 2010 @@ -12,11 +12,12 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.debug import debug_print, fatalerror +from pypy.rlib.debug import debug_start, debug_stop from pypy.rpython.lltypesystem.lloperation import llop from pypy.translator.simplify import get_funcobj, get_functype from pypy.translator.unsimplify import call_final_function -from pypy.jit.metainterp import history, pyjitpl, gc +from pypy.jit.metainterp import history, pyjitpl, gc, memmgr from pypy.jit.metainterp.pyjitpl import MetaInterpStaticData, MetaInterp from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.jitprof import Profiler, EmptyProfiler @@ -24,21 +25,17 @@ from pypy.jit.metainterp.jitdriver import JitDriverStaticData from pypy.jit.codewriter import support, codewriter from pypy.jit.codewriter.policy import JitPolicy -from pypy.rlib.jit import DEBUG_STEPS, DEBUG_DETAILED, DEBUG_OFF, DEBUG_PROFILE # ____________________________________________________________ # Bootstrapping -def apply_jit(translator, backend_name="auto", debug_level=DEBUG_STEPS, - inline=False, - **kwds): +def apply_jit(translator, backend_name="auto", inline=False, **kwds): if 'CPUClass' not in kwds: from pypy.jit.backend.detect_cpu import getcpuclass kwds['CPUClass'] = getcpuclass(backend_name) - if debug_level > DEBUG_OFF: - ProfilerClass = Profiler - else: - ProfilerClass = EmptyProfiler + ProfilerClass = Profiler + # Always use Profiler here, which should have a very low impact. + # Otherwise you can try with ProfilerClass = EmptyProfiler. warmrunnerdesc = WarmRunnerDesc(translator, translate_support_code=True, listops=True, @@ -47,7 +44,6 @@ **kwds) for jd in warmrunnerdesc.jitdrivers_sd: jd.warmstate.set_param_inlining(inline) - jd.warmstate.set_param_debug(debug_level) warmrunnerdesc.finish() translator.warmrunnerdesc = warmrunnerdesc # for later debugging @@ -66,7 +62,7 @@ def jittify_and_run(interp, graph, args, repeat=1, backendopt=False, trace_limit=sys.maxint, - debug_level=DEBUG_STEPS, inline=False, **kwds): + inline=False, loop_longevity=0, **kwds): from pypy.config.config import ConfigError translator = interp.typer.annotator.translator try: @@ -83,7 +79,7 @@ jd.warmstate.set_param_trace_eagerness(2) # for tests jd.warmstate.set_param_trace_limit(trace_limit) jd.warmstate.set_param_inlining(inline) - jd.warmstate.set_param_debug(debug_level) + jd.warmstate.set_param_loop_longevity(loop_longevity) warmrunnerdesc.finish() res = interp.eval_graph(graph, args) if not kwds.get('translate_support_code', False): @@ -148,9 +144,11 @@ class WarmRunnerDesc(object): def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, - optimizer=None, ProfilerClass=EmptyProfiler, **kwds): + optimizer=None, ProfilerClass=EmptyProfiler, + jit_ffi=None, **kwds): pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) + self.memory_manager = memmgr.MemoryManager() self.build_cpu(CPUClass, **kwds) self.find_portals() self.codewriter = codewriter.CodeWriter(self.cpu, self.jitdrivers_sd) @@ -162,8 +160,10 @@ self.check_access_directly_sanity(graphs) if backendopt: self.prejit_optimizations(policy, graphs) + elif self.opt.listops: + self.prejit_optimizations_minimal_inline(policy, graphs) - self.build_meta_interp(ProfilerClass) + self.build_meta_interp(ProfilerClass, jit_ffi) self.make_args_specifications() # from pypy.jit.metainterp.virtualref import VirtualRefInfo @@ -259,6 +259,10 @@ remove_asserts=True, really_remove_asserts=True) + def prejit_optimizations_minimal_inline(self, policy, graphs): + from pypy.translator.backendopt.inline import auto_inline_graphs + auto_inline_graphs(self.translator, graphs, 0.01) + def build_cpu(self, CPUClass, translate_support_code=False, no_stats=False, **kwds): assert CPUClass is not None @@ -277,11 +281,14 @@ translate_support_code, gcdescr=self.gcdescr) self.cpu = cpu - def build_meta_interp(self, ProfilerClass): + def build_meta_interp(self, ProfilerClass, jit_ffi=None): + if jit_ffi is None: + jit_ffi = self.translator.config.translation.jit_ffi self.metainterp_sd = MetaInterpStaticData(self.cpu, self.opt, ProfilerClass=ProfilerClass, - warmrunnerdesc=self) + warmrunnerdesc=self, + jit_ffi=jit_ffi) def make_virtualizable_infos(self): vinfos = {} @@ -713,7 +720,7 @@ loop_token = fail_descr.handle_fail(self.metainterp_sd, jd) except JitException, e: return handle_jitexception(e) - fail_descr = self.cpu.execute_token(loop_token) + fail_descr = self.execute_token(loop_token) jd._assembler_call_helper = assembler_call_helper # for debugging jd._assembler_helper_ptr = self.helper_func( @@ -807,3 +814,10 @@ py.test.skip("rewrite_force_virtual: port it to ootype") all_graphs = self.translator.graphs vrefinfo.replace_force_virtual_with_call(all_graphs) + + # ____________________________________________________________ + + def execute_token(self, loop_token): + fail_descr = self.cpu.execute_token(loop_token) + self.memory_manager.keep_loop_alive(loop_token) + return fail_descr Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/warmstate.py Sat Dec 11 15:10:15 2010 @@ -1,4 +1,4 @@ -import sys +import sys, weakref from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rffi from pypy.rpython.ootypesystem import ootype from pypy.rpython.annlowlevel import hlstr, llstr, cast_base_ptr_to_instance @@ -9,7 +9,6 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.jit import (PARAMETERS, OPTIMIZER_SIMPLE, OPTIMIZER_FULL, OPTIMIZER_NO_UNROLL) -from pypy.rlib.jit import DEBUG_PROFILE from pypy.rlib.jit import BaseJitCell from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.jit.metainterp import history @@ -150,9 +149,34 @@ # counter == -1: there is an entry bridge for this cell # counter == -2: tracing is currently going on for this cell counter = 0 - compiled_merge_points = None + compiled_merge_points_wref = None # list of weakrefs to LoopToken dont_trace_here = False - entry_loop_token = None + wref_entry_loop_token = None # (possibly) one weakref to LoopToken + + def get_compiled_merge_points(self): + result = [] + if self.compiled_merge_points_wref is not None: + for wref in self.compiled_merge_points_wref: + looptoken = wref() + if looptoken is not None: + result.append(looptoken) + return result + + def set_compiled_merge_points(self, looptokens): + self.compiled_merge_points_wref = [self._makeref(token) + for token in looptokens] + + def get_entry_loop_token(self): + if self.wref_entry_loop_token is not None: + return self.wref_entry_loop_token() + return None + + def set_entry_loop_token(self, looptoken): + self.wref_entry_loop_token = self._makeref(looptoken) + + def _makeref(self, looptoken): + assert looptoken is not None + return weakref.ref(looptoken) # ____________________________________________________________ @@ -165,6 +189,8 @@ "NOT_RPYTHON" self.warmrunnerdesc = warmrunnerdesc self.jitdriver_sd = jitdriver_sd + if warmrunnerdesc is not None: # for tests + self.cpu = warmrunnerdesc.cpu try: self.profiler = warmrunnerdesc.metainterp_sd.profiler except AttributeError: # for tests @@ -176,7 +202,7 @@ meth(default_value) def set_param_threshold(self, threshold): - if threshold < 0: + if threshold <= 0: self.increment_threshold = 0 # never reach the THRESHOLD_LIMIT return if threshold < 2: @@ -209,10 +235,11 @@ else: raise ValueError("unknown optimizer") - def set_param_debug(self, value): - self.debug_level = value - if self.profiler is not None: - self.profiler.set_printing(value >= DEBUG_PROFILE) + def set_param_loop_longevity(self, value): + # note: it's a global parameter, not a per-jitdriver one + if (self.warmrunnerdesc is not None and + self.warmrunnerdesc.memory_manager is not None): # all for tests + self.warmrunnerdesc.memory_manager.set_max_age(value) def disable_noninlinable_function(self, greenkey): cell = self.jit_cell_at_key(greenkey) @@ -225,12 +252,15 @@ def attach_unoptimized_bridge_from_interp(self, greenkey, entry_loop_token): cell = self.jit_cell_at_key(greenkey) - cell.counter = -1 - old_token = cell.entry_loop_token - cell.entry_loop_token = entry_loop_token + old_token = cell.get_entry_loop_token() + cell.set_entry_loop_token(entry_loop_token) + cell.counter = -1 # valid entry bridge attached if old_token is not None: - cpu = self.warmrunnerdesc.cpu - cpu.redirect_call_assembler(old_token, entry_loop_token) + self.cpu.redirect_call_assembler(old_token, entry_loop_token) + # entry_loop_token is also kept alive by any loop that used + # to point to old_token. Actually freeing old_token early + # is a pointless optimization (it is tiny). + old_token.record_jump_to(entry_loop_token) # ---------- @@ -239,7 +269,8 @@ if hasattr(self, 'maybe_compile_and_run'): return self.maybe_compile_and_run - metainterp_sd = self.warmrunnerdesc.metainterp_sd + warmrunnerdesc = self.warmrunnerdesc + metainterp_sd = warmrunnerdesc.metainterp_sd jitdriver_sd = self.jitdriver_sd vinfo = jitdriver_sd.virtualizable_info index_of_virtualizable = jitdriver_sd.index_of_virtualizable @@ -297,23 +328,27 @@ assert cell.counter == -1 if not confirm_enter_jit(*args): return + loop_token = cell.get_entry_loop_token() + if loop_token is None: # it was a weakref that has been freed + cell.counter = 0 + return # machine code was already compiled for these greenargs # get the assembler and fill in the boxes set_future_values(*args[num_green_args:]) - loop_token = cell.entry_loop_token # ---------- execute assembler ---------- while True: # until interrupted by an exception metainterp_sd.profiler.start_running() debug_start("jit-running") - fail_descr = metainterp_sd.cpu.execute_token(loop_token) + fail_descr = warmrunnerdesc.execute_token(loop_token) debug_stop("jit-running") metainterp_sd.profiler.end_running() + loop_token = None # for test_memmgr if vinfo is not None: vinfo.reset_vable_token(virtualizable) loop_token = fail_descr.handle_fail(metainterp_sd, jitdriver_sd) - + maybe_compile_and_run._dont_inline_ = True self.maybe_compile_and_run = maybe_compile_and_run return maybe_compile_and_run @@ -459,7 +494,7 @@ warmrunnerdesc = self.warmrunnerdesc jitdriver_sd = self.jitdriver_sd - cpu = warmrunnerdesc.cpu + cpu = self.cpu vinfo = jitdriver_sd.virtualizable_info red_args_types = unrolling_iterable(jitdriver_sd._red_args_types) # @@ -508,10 +543,11 @@ if hasattr(self, 'get_location_str'): return # + warmrunnerdesc = self.warmrunnerdesc unwrap_greenkey = self.make_unwrap_greenkey() jit_getter = self.make_jitcell_getter() jd = self.jitdriver_sd - cpu = self.warmrunnerdesc.cpu + cpu = self.cpu def can_inline_greenargs(*greenargs): if can_never_inline(*greenargs): @@ -529,11 +565,16 @@ def get_assembler_token(greenkey, redboxes): # 'redboxes' is only used to know the types of red arguments cell = self.jit_cell_at_key(greenkey) - if cell.entry_loop_token is None: + entry_loop_token = cell.get_entry_loop_token() + if entry_loop_token is None: from pypy.jit.metainterp.compile import compile_tmp_callback - cell.entry_loop_token = compile_tmp_callback(cpu, jd, greenkey, - redboxes) - return cell.entry_loop_token + if cell.counter == -1: # used to be a valid entry bridge, + cell.counter = 0 # but was freed in the meantime. + memmgr = warmrunnerdesc.memory_manager + entry_loop_token = compile_tmp_callback(cpu, jd, greenkey, + redboxes, memmgr) + cell.set_entry_loop_token(entry_loop_token) + return entry_loop_token self.get_assembler_token = get_assembler_token # Modified: pypy/branch/jit-unroll-loops/pypy/jit/tl/spli/interpreter.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/tl/spli/interpreter.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/tl/spli/interpreter.py Sat Dec 11 15:10:15 2010 @@ -105,26 +105,32 @@ self.stack_depth += 1 def pop(self): - self.stack_depth -= 1 - val = self.value_stack[self.stack_depth] - self.value_stack[self.stack_depth] = None + sd = self.stack_depth - 1 + assert sd >= 0 + self.stack_depth = sd + val = self.value_stack[sd] + self.value_stack[sd] = None return val def pop_many(self, n): return [self.pop() for i in range(n)] def peek(self): - return self.value_stack[self.stack_depth - 1] + sd = self.stack_depth - 1 + assert sd >= 0 + return self.value_stack[sd] def POP_TOP(self, _, next_instr, code): self.pop() return next_instr def LOAD_FAST(self, name_index, next_instr, code): + assert name_index >= 0 self.push(self.locals[name_index]) return next_instr def STORE_FAST(self, name_index, next_instr, code): + assert name_index >= 0 self.locals[name_index] = self.pop() return next_instr Modified: pypy/branch/jit-unroll-loops/pypy/jit/tl/tl.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/tl/tl.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/tl/tl.py Sat Dec 11 15:10:15 2010 @@ -16,32 +16,40 @@ def __init__(self, size): self = hint(self, access_directly=True, fresh_virtualizable=True) self.stack = [0] * size - self.stackpos = 0 + self.stackpos = 0 # always store a known-nonneg integer here def append(self, elem): self.stack[self.stackpos] = elem self.stackpos += 1 def pop(self): - self.stackpos -= 1 - if self.stackpos < 0: + stackpos = self.stackpos - 1 + if stackpos < 0: raise IndexError - return self.stack[self.stackpos] + self.stackpos = stackpos # always store a known-nonneg integer here + return self.stack[stackpos] def pick(self, i): - self.append(self.stack[self.stackpos - i - 1]) + n = self.stackpos - i - 1 + assert n >= 0 + self.append(self.stack[n]) def put(self, i): elem = self.pop() - self.stack[self.stackpos - i - 1] = elem + n = self.stackpos - i - 1 + assert n >= 0 + self.stack[n] = elem def roll(self, r): if r < -1: i = self.stackpos + r if i < 0: raise IndexError - elem = self.stack[self.stackpos - 1] + n = self.stackpos - 1 + assert n >= 0 + elem = self.stack[n] for j in range(self.stackpos - 2, i - 1, -1): + assert j >= 0 self.stack[j + 1] = self.stack[j] self.stack[i] = elem elif r > 1: @@ -51,7 +59,9 @@ elem = self.stack[i] for j in range(i, self.stackpos - 1): self.stack[j] = self.stack[j + 1] - self.stack[self.stackpos - 1] = elem + n = self.stackpos - 1 + assert n >= 0 + self.stack[n] = elem def make_interp(supports_call): Modified: pypy/branch/jit-unroll-loops/pypy/jit/tool/jitoutput.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/tool/jitoutput.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/tool/jitoutput.py Sat Dec 11 15:10:15 2010 @@ -27,6 +27,10 @@ (('nvirtuals',), '^nvirtuals:\s+(\d+)$'), (('nvholes',), '^nvholes:\s+(\d+)$'), (('nvreused',), '^nvreused:\s+(\d+)$'), + (('total_compiled_loops',), '^Total # of loops:\s+(\d+)$'), + (('total_compiled_bridges',), '^Total # of bridges:\s+(\d+)$'), + (('total_freed_loops',), '^Freed # of loops:\s+(\d+)$'), + (('total_freed_bridges',), '^Freed # of bridges:\s+(\d+)$'), ] class Ops(object): Modified: pypy/branch/jit-unroll-loops/pypy/jit/tool/loopcounter.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/tool/loopcounter.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/tool/loopcounter.py Sat Dec 11 15:10:15 2010 @@ -7,32 +7,41 @@ import py import sys import optparse +import re def get_timestamp(line): - import re match = re.match(r'\[([0-9a-f]*)\] .*', line) return int(match.group(1), 16) -def main(logfile, options): - log = open(logfile) +def count_loops_and_bridges(log): loops = 0 bridges = 0 time0 = None - print 'timestamp,total,loops,bridges' - for line in log: + lines = iter(log) + for line in lines: if time0 is None and line.startswith('['): time0 = get_timestamp(line) - if '{jit-log-opt-' in line: - time_now = get_timestamp(line) - if '{jit-log-opt-loop' in line: + if '{jit-mem-looptoken-' in line: + time_now = get_timestamp(line) - time0 + text = lines.next() + if text.startswith('allocating Loop #'): loops += 1 - elif '{jit-log-opt-bridge' in line: + elif text.startswith('allocating Bridge #'): bridges += 1 + elif text.startswith('freeing Loop #'): + match = re.match('freeing Loop # .* with ([0-9]*) attached bridges\n', text) + loops -=1 + bridges -= int(match.group(1)) else: - assert False, 'unknown category %s' % line + assert False, 'unknown line' % line total = loops+bridges - timestamp = time_now - time0 - print '%d,%d,%d,%d' % (timestamp, total, loops, bridges) + yield (time_now, total, loops, bridges) + +def main(logfile, options): + print 'timestamp,total,loops,bridges' + log = open(logfile) + for timestamp, total, loops, bridges in count_loops_and_bridges(log): + print '%d,%d,%d,%d' % (timestamp, total, loops, bridges) if __name__ == '__main__': parser = optparse.OptionParser(usage="%prog loopfile [options]") @@ -40,5 +49,4 @@ if len(args) != 1: parser.print_help() sys.exit(2) - main(args[0], options) Modified: pypy/branch/jit-unroll-loops/pypy/jit/tool/test/test_jitoutput.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/tool/test/test_jitoutput.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/tool/test/test_jitoutput.py Sat Dec 11 15:10:15 2010 @@ -1,10 +1,11 @@ import py from pypy.jit.metainterp.warmspot import ll_meta_interp -from pypy.rlib.jit import JitDriver, DEBUG_PROFILE +from pypy.rlib.jit import JitDriver from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.jitprof import Profiler, JITPROF_LINES from pypy.jit.tool.jitoutput import parse_prof +from pypy.tool.logparser import parse_log, extract_category def test_really_run(): """ This test checks whether output of jitprof did not change. @@ -21,13 +22,15 @@ cap = py.io.StdCaptureFD() try: ll_meta_interp(f, [10], CPUClass=runner.LLtypeCPU, type_system='lltype', - ProfilerClass=Profiler, debug_level=DEBUG_PROFILE) + ProfilerClass=Profiler) finally: out, err = cap.reset() - err = "\n".join(err.splitlines()[-JITPROF_LINES:]) - print err - assert err.count("\n") == JITPROF_LINES - 1 - info = parse_prof(err) + + log = parse_log(err.splitlines(True)) + err_sections = list(extract_category(log, 'jit-summary')) + [err1] = err_sections # there should be exactly one jit-summary + assert err1.count("\n") == JITPROF_LINES + info = parse_prof(err1) # assert did not crash # asserts below are a bit delicate, possibly they might be deleted assert info.tracing_no == 1 @@ -60,6 +63,10 @@ nvirtuals: 13 nvholes: 14 nvreused: 15 +Total # of loops: 100 +Total # of bridges: 300 +Freed # of loops: 99 +Freed # of bridges: 299 ''' def test_parse(): Modified: pypy/branch/jit-unroll-loops/pypy/module/__pypy__/__init__.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/__pypy__/__init__.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/__pypy__/__init__.py Sat Dec 11 15:10:15 2010 @@ -11,6 +11,11 @@ 'internal_repr' : 'interp_magic.internal_repr', 'bytebuffer' : 'bytebuffer.bytebuffer', 'identity_dict' : 'interp_identitydict.W_IdentityDict', + 'debug_start' : 'interp_debug.debug_start', + 'debug_print' : 'interp_debug.debug_print', + 'debug_stop' : 'interp_debug.debug_stop', + 'debug_print_once' : 'interp_debug.debug_print_once', + 'builtinify' : 'interp_magic.builtinify', } def setup_after_space_initialization(self): Modified: pypy/branch/jit-unroll-loops/pypy/module/__pypy__/interp_magic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/__pypy__/interp_magic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/__pypy__/interp_magic.py Sat Dec 11 15:10:15 2010 @@ -51,3 +51,9 @@ return space.newtuple([space.newint(cache.hits.get(name, 0)), space.newint(cache.misses.get(name, 0))]) mapdict_cache_counter.unwrap_spec = [ObjSpace, str] + +def builtinify(space, w_func): + from pypy.interpreter.function import Function, BuiltinFunction + func = space.interp_w(Function, w_func) + bltn = BuiltinFunction(func) + return space.wrap(bltn) Modified: pypy/branch/jit-unroll-loops/pypy/module/__pypy__/test/test_special.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/__pypy__/test/test_special.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/__pypy__/test/test_special.py Sat Dec 11 15:10:15 2010 @@ -21,3 +21,18 @@ def test_cpumodel(self): import __pypy__ assert hasattr(__pypy__, 'cpumodel') + + def test_builtinify(self): + import __pypy__ + class A(object): + a = lambda *args: args + b = __pypy__.builtinify(a) + my = A() + assert my.a() == (my,) + assert my.b() == () + assert A.a(my) == (my,) + assert A.b(my) == (my,) + assert A.a.im_func(my) == (my,) + assert not hasattr(A.b, 'im_func') + assert A.a is not A.__dict__['a'] + assert A.b is A.__dict__['b'] Modified: pypy/branch/jit-unroll-loops/pypy/module/_lsprof/interp_lsprof.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/_lsprof/interp_lsprof.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/_lsprof/interp_lsprof.py Sat Dec 11 15:10:15 2010 @@ -163,8 +163,11 @@ if isinstance(w_arg, Method): w_function = w_arg.w_function class_name = w_arg.w_class.getname(space, '?') - assert isinstance(w_function, Function) - return "{method '%s' of '%s' objects}" % (w_function.name, class_name) + if isinstance(w_function, Function): + name = w_function.name + else: + name = '?' + return "{method '%s' of '%s' objects}" % (name, class_name) elif isinstance(w_arg, Function): if w_arg.w_module is None: module = '' @@ -176,7 +179,8 @@ module += '.' return '{%s%s}' % (module, w_arg.name) else: - return '{!!!unknown!!!}' + class_name = space.type(w_arg).getname(space, '?') + return "{'%s' object}" % (class_name,) def lsprof_call(space, w_self, frame, event, w_arg): assert isinstance(w_self, W_Profiler) Modified: pypy/branch/jit-unroll-loops/pypy/module/_lsprof/test/test_cprofile.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/_lsprof/test/test_cprofile.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/_lsprof/test/test_cprofile.py Sat Dec 11 15:10:15 2010 @@ -2,9 +2,10 @@ from pypy.conftest import gettestobjspace, option class AppTestCProfile(object): + keywords = {} def setup_class(cls): - space = gettestobjspace(usemodules=('_lsprof',)) + space = gettestobjspace(usemodules=('_lsprof',), **cls.keywords) cls.w_expected_output = space.wrap(expected_output) cls.space = space cls.w_file = space.wrap(__file__) @@ -148,6 +149,12 @@ finally: sys.path.pop(0) + +class AppTestWithDifferentBytecodes(AppTestCProfile): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True, + 'objspace.opcodes.CALL_METHOD': True} + + expected_output = {} expected_output['print_stats'] = """\ 126 function calls (106 primitive calls) in 1.000 CPU seconds @@ -165,11 +172,11 @@ 2 0.000 0.000 0.140 0.070 profilee.py:84(helper2_indirect) 8 0.312 0.039 0.400 0.050 profilee.py:88(helper2) 8 0.064 0.008 0.080 0.010 profilee.py:98(subhelper) - 4 0.000 0.000 0.000 0.000 {.*append.*} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} 1 0.000 0.000 0.000 0.000 {.*disable.*} - 12 0.000 0.000 0.012 0.001 {hasattr.*} - 8 0.000 0.000 0.000 0.000 {range.*} - 4 0.000 0.000 0.000 0.000 {sys.exc_info.*} + 12 0.000 0.000 0.012 0.001 {hasattr} + 8 0.000 0.000 0.000 0.000 {range} + 4 0.000 0.000 0.000 0.000 {sys.exc_info} """ Modified: pypy/branch/jit-unroll-loops/pypy/module/_minimal_curses/__init__.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/_minimal_curses/__init__.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/_minimal_curses/__init__.py Sat Dec 11 15:10:15 2010 @@ -4,7 +4,7 @@ try: import _minimal_curses as _curses # when running on top of pypy-c except ImportError: - import py; py.test.skip("no _curses module") # no _curses at all + raise ImportError("no _curses or _minimal_curses module") # no _curses at all from pypy.interpreter.mixedmodule import MixedModule from pypy.module._minimal_curses import fficurses Modified: pypy/branch/jit-unroll-loops/pypy/module/_pickle_support/maker.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/_pickle_support/maker.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/_pickle_support/maker.py Sat Dec 11 15:10:15 2010 @@ -67,11 +67,12 @@ return space.wrap(tb) traceback_new.unwrap_spec = [ObjSpace] -def generator_new(space, frame, running): +def generator_new(space, w_frame, running): + frame = space.interp_w(PyFrame, w_frame, can_be_None=True) new_generator = GeneratorIterator(frame) new_generator.running = running return space.wrap(new_generator) -generator_new.unwrap_spec = [ObjSpace, PyFrame, int] +generator_new.unwrap_spec = [ObjSpace, W_Root, int] def xrangeiter_new(space, current, remaining, step): from pypy.module.__builtin__.functional import W_XRangeIterator Modified: pypy/branch/jit-unroll-loops/pypy/module/_stackless/interp_coroutine.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/_stackless/interp_coroutine.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/_stackless/interp_coroutine.py Sat Dec 11 15:10:15 2010 @@ -304,16 +304,19 @@ def w_descr__framestack(space, self): assert isinstance(self, AppCoroutine) - index = self.subctx.framestackdepth - if not index: - return space.newtuple([]) - items = [None] * index + counter = 0 f = self.subctx.topframe - while index > 0: - index -= 1 - items[index] = space.wrap(f) + while f is not None: + counter += 1 f = f.f_backref() - assert f is None + items = [None] * counter + f = self.subctx.topframe + while f is not None: + counter -= 1 + assert counter >= 0 + items[counter] = space.wrap(f) + f = f.f_backref() + assert counter == 0 return space.newtuple(items) def makeStaticMethod(module, classname, funcname): Modified: pypy/branch/jit-unroll-loops/pypy/module/array/test/test_array.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/array/test/test_array.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/array/test/test_array.py Sat Dec 11 15:10:15 2010 @@ -64,6 +64,7 @@ raises(TypeError, self.array, tc, None) def test_value_range(self): + import sys values = (-129, 128, -128, 127, 0, 255, -1, 256, -32768, 32767, -32769, 32768, 65535, 65536, -2147483647, -2147483648, 2147483647, 4294967295, 4294967296, @@ -88,7 +89,12 @@ a.append(v) for i, v in enumerate(ok * 2): assert a[i] == v - assert type(a[i]) is pt + assert type(a[i]) is pt or ( + # A special case: we return ints in Array('I') on 64-bits, + # whereas CPython returns longs. The difference is + # probably acceptable. + tc == 'I' and + sys.maxint > 2147483647 and type(a[i]) is int) for v in ok: a[1] = v assert a[0] == ok[0] Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/__init__.py Sat Dec 11 15:10:15 2010 @@ -69,6 +69,7 @@ import pypy.module.cpyext.weakrefobject import pypy.module.cpyext.funcobject import pypy.module.cpyext.classobject +import pypy.module.cpyext.pypyintf # now that all rffi_platform.Struct types are registered, configure them api.configure_types() Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/api.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/api.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/api.py Sat Dec 11 15:10:15 2010 @@ -321,6 +321,8 @@ 'PyCObject_Type', 'init_pycobject', 'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer', + + 'PyStructSequence_InitType', 'PyStructSequence_New', ] TYPES = {} GLOBALS = { # this needs to include all prebuilt pto, otherwise segfaults occur @@ -845,6 +847,7 @@ source_dir / "bufferobject.c", source_dir / "object.c", source_dir / "cobject.c", + source_dir / "structseq.c", ], separate_module_sources = [code, struct_source], export_symbols=export_symbols_eci, Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/cdatetime.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/cdatetime.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/cdatetime.py Sat Dec 11 15:10:15 2010 @@ -4,8 +4,7 @@ from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, cpython_struct, PyObjectFields) from pypy.module.cpyext.import_ import PyImport_Import -from pypy.module.cpyext.typeobject import PyTypeObjectPtr, render_immortal -from pypy.module.cpyext.state import State +from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_renamer @@ -25,48 +24,26 @@ datetimeAPI = lltype.malloc(PyDateTime_CAPI, flavor='raw', track_allocation=False) - if not we_are_translated(): - datetimeAPI_dealloc(space) - space.fromcache(State).datetimeAPI = datetimeAPI - w_datetime = PyImport_Import(space, space.wrap("datetime")) w_type = space.getattr(w_datetime, space.wrap("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DateType, w_type) w_type = space.getattr(w_datetime, space.wrap("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DateTimeType, w_type) w_type = space.getattr(w_datetime, space.wrap("time")) datetimeAPI.c_TimeType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_TimeType, w_type) w_type = space.getattr(w_datetime, space.wrap("timedelta")) datetimeAPI.c_DeltaType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) - render_immortal(datetimeAPI.c_DeltaType, w_type) return datetimeAPI -def datetimeAPI_dealloc(space): - "Used in tests only, to please the refcount checker" - if we_are_translated(): - return - datetimeAPI = space.fromcache(State).datetimeAPI - if datetimeAPI is None: - return - space.fromcache(State).datetimeAPI = None - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DateType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DateTimeType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_TimeType)) - Py_DecRef(space, rffi.cast(PyObject, datetimeAPI.c_DeltaType)) - lltype.free(datetimeAPI, flavor='raw') - PyDateTime_Date = PyObject PyDateTime_Time = PyObject PyDateTime_DateTime = PyObject Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/include/Python.h ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/include/Python.h (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/include/Python.h Sat Dec 11 15:10:15 2010 @@ -8,6 +8,8 @@ # include # include # include +# include +# include # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) # define PyAPI_FUNC(RTYPE) RTYPE # define PyAPI_DATA(RTYPE) extern RTYPE @@ -103,6 +105,7 @@ #include "sliceobject.h" #include "datetime.h" #include "pystate.h" +#include "fileobject.h" // XXX This shouldn't be included here #include "structmember.h" @@ -120,4 +123,8 @@ #define PyDoc_STR(str) "" #endif +/* PyPy does not implement --with-fpectl */ +#define PyFPE_START_PROTECT(err_string, leave_stmt) +#define PyFPE_END_PROTECT(v) + #endif Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/intobject.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/intobject.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/intobject.py Sat Dec 11 15:10:15 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import rffi, lltype +from pypy.interpreter.error import OperationError from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, build_type_checkers, Py_ssize_t) @@ -19,6 +20,9 @@ already one, and then return its value. If there is an error, -1 is returned, and the caller should check PyErr_Occurred() to find out whether there was an error, or whether the value just happened to be -1.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Unsigned, error=-1) @@ -26,6 +30,9 @@ """Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.uint_w(space.int(w_obj)) @cpython_api([PyObject], lltype.Signed, error=CANNOT_FAIL) @@ -39,6 +46,9 @@ PyLongObject, if it is not already one, and then return its value as Py_ssize_t. """ + if w_obj is None: + raise OperationError(space.w_TypeError, + space.wrap("an integer is required, got NULL")) return space.int_w(w_obj) # XXX this is wrong on win64 @cpython_api([Py_ssize_t], PyObject) @@ -48,4 +58,3 @@ returned. """ return space.wrap(ival) # XXX this is wrong on win64 - Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/object.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/object.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/object.py Sat Dec 11 15:10:15 2010 @@ -2,12 +2,13 @@ from pypy.module.cpyext.api import ( cpython_api, generic_cpy_call, CANNOT_FAIL, Py_ssize_t, Py_ssize_tP, PyVarObject, Py_TPFLAGS_HEAPTYPE, Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, - Py_GE, CONST_STRING, FILEP, fwrite) + Py_GE, CONST_STRING, FILEP, fwrite, build_type_checkers) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, track_reference, get_typedescr, RefcountState) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall +from pypy.module._file.interp_file import W_File from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import W_TypeObject from pypy.interpreter.error import OperationError @@ -428,6 +429,34 @@ rffi.free_nonmovingbuffer(data, buf) return 0 +PyFile_Check, PyFile_CheckExact = build_type_checkers("File", W_File) + + at cpython_api([PyObject, rffi.INT_real], PyObject) +def PyFile_GetLine(space, w_obj, n): + """ + Equivalent to p.readline([n]), this function reads one line from the + object p. p may be a file object or any object with a readline() + method. If n is 0, exactly one line is read, regardless of the length of + the line. If n is greater than 0, no more than n bytes will be read + from the file; a partial line can be returned. In both cases, an empty string + is returned if the end of the file is reached immediately. If n is less than + 0, however, one line is read regardless of length, but EOFError is + raised if the end of the file is reached immediately.""" + try: + w_readline = space.getattr(w_obj, space.wrap('readline')) + except OperationError: + raise OperationError( + space.w_TypeError, space.wrap( + "argument must be a file, or have a readline() method.")) + + n = rffi.cast(lltype.Signed, n) + if space.is_true(space.gt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline, space.wrap(n)) + elif space.is_true(space.lt(space.wrap(n), space.wrap(0))): + return space.call_function(w_readline) + else: + # XXX Raise EOFError as specified + return space.call_function(w_readline) @cpython_api([CONST_STRING, CONST_STRING], PyObject) def PyFile_FromString(space, filename, mode): """ @@ -437,4 +466,3 @@ w_filename = space.wrap(rffi.charp2str(filename)) w_mode = space.wrap(rffi.charp2str(mode)) return space.call_method(space.builtin, 'file', w_filename, w_mode) - Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/pyerrors.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/pyerrors.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/pyerrors.py Sat Dec 11 15:10:15 2010 @@ -5,7 +5,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, Py_DecRef, borrow_from) + PyObject, PyObjectP, make_ref, from_ref, Py_DecRef, borrow_from) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from pypy.rlib.rposix import get_errno @@ -80,6 +80,21 @@ Py_DecRef(space, w_value) Py_DecRef(space, w_traceback) + at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) +def PyErr_NormalizeException(space, exc_p, val_p, tb_p): + """Under certain circumstances, the values returned by PyErr_Fetch() below + can be "unnormalized", meaning that *exc is a class object but *val is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance.""" + operr = OperationError(from_ref(space, exc_p[0]), + from_ref(space, val_p[0])) + operr.normalize_exception(space) + Py_DecRef(space, exc_p[0]) + Py_DecRef(space, val_p[0]) + exc_p[0] = make_ref(space, operr.w_type) + val_p[0] = make_ref(space, operr.get_w_value(space)) + @cpython_api([], lltype.Void) def PyErr_BadArgument(space): """This is a shorthand for PyErr_SetString(PyExc_TypeError, message), where @@ -114,10 +129,29 @@ function around a system call can write return PyErr_SetFromErrno(type); when the system call returns an error. Return value: always NULL.""" + PyErr_SetFromErrnoWithFilename(space, w_type, + lltype.nullptr(rffi.CCHARP.TO)) + + at cpython_api([PyObject, rffi.CCHARP], PyObject) +def PyErr_SetFromErrnoWithFilename(space, w_type, llfilename): + """Similar to PyErr_SetFromErrno(), with the additional behavior that if + filename is not NULL, it is passed to the constructor of type as a third + parameter. In the case of exceptions such as IOError and OSError, + this is used to define the filename attribute of the exception instance. + Return value: always NULL.""" # XXX Doesn't actually do anything with PyErr_CheckSignals. errno = get_errno() msg = os.strerror(errno) - w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg)) + if llfilename: + w_filename = rffi.charp2str(llfilename) + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg), + space.wrap(w_filename)) + else: + w_error = space.call_function(w_type, + space.wrap(errno), + space.wrap(msg)) raise OperationError(w_type, w_error) @cpython_api([], rffi.INT_real, error=-1) Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/pythonrun.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/pythonrun.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/pythonrun.py Sat Dec 11 15:10:15 2010 @@ -1,6 +1,16 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL +from pypy.module.cpyext.state import State @cpython_api([], rffi.INT_real, error=CANNOT_FAIL) def Py_IsInitialized(space): return 1 + + at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) +def Py_GetProgramName(space): + """ + Return the program name set with Py_SetProgramName(), or the default. + The returned string points into static storage; the caller should not modify its + value.""" + return space.fromcache(State).get_programname() + Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/sequence.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/sequence.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/sequence.py Sat Dec 11 15:10:15 2010 @@ -116,6 +116,14 @@ This is the equivalent of the Python expression o1 + o2.""" return space.add(w_o1, w_o2) + at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) +def PySequence_Contains(space, w_obj, w_value): + """Determine if o contains value. If an item in o is equal to value, + return 1, otherwise return 0. On error, return -1. This is + equivalent to the Python expression value in o.""" + w_res = space.contains(w_obj, w_value) + return space.int_w(w_res) + @cpython_api([PyObject], PyObject) def PySeqIter_New(space, w_seq): """Return an iterator that works with a general sequence object, seq. The Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/slotdefs.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/slotdefs.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/slotdefs.py Sat Dec 11 15:10:15 2010 @@ -6,7 +6,7 @@ unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc, getattrfunc, setattrofunc, lenfunc, ssizeargfunc, ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc, hashfunc, - descrgetfunc, descrsetfunc) + descrgetfunc, descrsetfunc, objobjproc) from pypy.module.cpyext.pyobject import from_ref from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.state import State @@ -156,6 +156,15 @@ if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) +def wrap_objobjproc(space, w_self, w_args, func): + func_target = rffi.cast(objobjproc, func) + check_num_args(space, w_args, 1) + w_value, = space.fixedview(w_args) + res = generic_cpy_call(space, func_target, w_self, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.wrap(res) + def wrap_ssizessizeargfunc(space, w_self, w_args, func): func_target = rffi.cast(ssizessizeargfunc, func) check_num_args(space, w_args, 2) Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/src/getargs.c Sat Dec 11 15:10:15 2010 @@ -453,6 +453,7 @@ strncpy(msgbuf, "is not retrievable", bufsize); return msgbuf; } + PyPy_Borrow(arg, item); msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); /* PySequence_GetItem calls tp->sq_item, which INCREFs */ Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/state.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/state.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/state.py Sat Dec 11 15:10:15 2010 @@ -5,11 +5,10 @@ class State: - datetimeAPI = None # used in tests - def __init__(self, space): self.space = space self.reset() + self.programname = lltype.nullptr(rffi.CCHARP.TO) def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef @@ -43,3 +42,16 @@ if always: raise OperationError(self.space.w_SystemError, self.space.wrap( "Function returned an error result without setting an exception")) + + def get_programname(self): + if not self.programname: + space = self.space + argv = space.sys.get('argv') + if space.int_w(space.len(argv)): + argv0 = space.getitem(argv, space.wrap(0)) + progname = space.str_w(argv0) + else: + progname = "pypy" + self.programname = rffi.str2charp(progname) + lltype.render_immortal(self.programname) + return self.programname Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/stubs.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/stubs.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/stubs.py Sat Dec 11 15:10:15 2010 @@ -668,24 +668,6 @@ """ raise NotImplementedError - at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void) -def PyErr_NormalizeException(space, exc, val, tb): - """Under certain circumstances, the values returned by PyErr_Fetch() below - can be "unnormalized", meaning that *exc is a class object but *val is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance.""" - raise NotImplementedError - - at cpython_api([PyObject, rffi.CCHARP], PyObject) -def PyErr_SetFromErrnoWithFilename(space, type, filename): - """Similar to PyErr_SetFromErrno(), with the additional behavior that if - filename is not NULL, it is passed to the constructor of type as a third - parameter. In the case of exceptions such as IOError and OSError, - this is used to define the filename attribute of the exception instance. - Return value: always NULL.""" - raise NotImplementedError - @cpython_api([rffi.INT_real], PyObject) def PyErr_SetFromWindowsErr(space, ierr): """This is a convenience function to raise WindowsError. If called with @@ -809,21 +791,6 @@ successful invocation of Py_EnterRecursiveCall().""" raise NotImplementedError - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_Check(space, p): - """Return true if its argument is a PyFileObject or a subtype of - PyFileObject. - - Allowed subtypes to be accepted.""" - raise NotImplementedError - - at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) -def PyFile_CheckExact(space, p): - """Return true if its argument is a PyFileObject, but not a subtype of - PyFileObject. - """ - raise NotImplementedError - @cpython_api([FILE, rffi.CCHARP, rffi.CCHARP, rffi.INT_real], PyObject) def PyFile_FromFile(space, fp, name, mode, close): """Create a new PyFileObject from the already-open standard C file @@ -857,22 +824,6 @@ """ raise NotImplementedError - at cpython_api([PyObject, rffi.INT_real], PyObject) -def PyFile_GetLine(space, p, n): - """ - - - - Equivalent to p.readline([n]), this function reads one line from the - object p. p may be a file object or any object with a readline() - method. If n is 0, exactly one line is read, regardless of the length of - the line. If n is greater than 0, no more than n bytes will be read - from the file; a partial line can be returned. In both cases, an empty string - is returned if the end of the file is reached immediately. If n is less than - 0, however, one line is read regardless of length, but EOFError is - raised if the end of the file is reached immediately.""" - raise NotImplementedError - @cpython_api([PyObject], PyObject) def PyFile_Name(space, p): """Return the name of the file specified by p as a string object.""" @@ -1431,17 +1382,6 @@ raise NotImplementedError @cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) -def Py_GetProgramName(space, ): - """ - - - - Return the program name set with Py_SetProgramName(), or the default. - The returned string points into static storage; the caller should not modify its - value.""" - raise NotImplementedError - - at cpython_api([], rffi.CCHARP, error=CANNOT_FAIL) def Py_GetPrefix(space, ): """Return the prefix for installed platform-independent files. This is derived through a number of complicated rules from the program name set with @@ -2289,13 +2229,6 @@ in your code for properly supporting 64-bit systems.""" raise NotImplementedError - at cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) -def PySequence_Contains(space, o, value): - """Determine if o contains value. If an item in o is equal to value, - return 1, otherwise return 0. On error, return -1. This is - equivalent to the Python expression value in o.""" - raise NotImplementedError - @cpython_api([PyObject, PyObject], Py_ssize_t, error=-1) def PySequence_Index(space, o, value): """Return the first index i for which o[i] == value. On error, return Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_arraymodule.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_arraymodule.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_arraymodule.py Sat Dec 11 15:10:15 2010 @@ -14,6 +14,7 @@ arr.append(4) assert arr.tolist() == [1, 2, 3, 4] assert len(arr) == 4 + self.cleanup_references() def test_iter(self): module = self.import_module(name='array') @@ -22,6 +23,7 @@ for i in arr: sum += i assert sum == 6 + self.cleanup_references() def test_index(self): module = self.import_module(name='array') @@ -32,6 +34,7 @@ assert arr.tolist() == [1,2,4] arr[2] = 99 assert arr.tolist() == [1,2,99] + self.cleanup_references() def test_slice_get(self): module = self.import_module(name='array') @@ -40,3 +43,4 @@ assert arr[1:].tolist() == [2,3,4] assert arr[:2].tolist() == [1,2] assert arr[1:3].tolist() == [2,3] + self.cleanup_references() Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_cpyext.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_cpyext.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_cpyext.py Sat Dec 11 15:10:15 2010 @@ -1,10 +1,12 @@ import sys +import weakref import os.path import py from pypy.conftest import gettestobjspace from pypy.interpreter.error import OperationError +from pypy.interpreter.gateway import interp2app from pypy.rpython.lltypesystem import rffi, lltype, ll2ctypes from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.translator import platform @@ -79,6 +81,23 @@ self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) class LeakCheckingTest(object): + @staticmethod + def cleanup_references(space): + state = space.fromcache(RefcountState) + + import gc; gc.collect() + # Clear all lifelines, objects won't resurrect + for w_obj, obj in state.lifeline_dict._dict.items(): + if w_obj not in state.py_objects_w2r: + state.lifeline_dict.set(w_obj, None) + del obj + import gc; gc.collect() + + for w_obj in state.non_heaptypes_w: + Py_DecRef(space, w_obj) + state.non_heaptypes_w[:] = [] + state.reset_borrowed_references() + def check_and_print_leaks(self): # check for sane refcnts import gc @@ -89,13 +108,6 @@ lost_objects_w = identity_dict() lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys()) - # Clear all lifelines, objects won't resurrect - for w_obj, obj in state.lifeline_dict._dict.items(): - if w_obj not in state.py_objects_w2r: - state.lifeline_dict.set(w_obj, None) - del obj - gc.collect() - for w_obj, obj in state.py_objects_w2r.iteritems(): base_refcnt = self.frozen_refcounts.get(w_obj) delta = obj.c_ob_refcnt @@ -105,7 +117,12 @@ if delta != 0: leaking = True print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta) - lifeline = state.lifeline_dict.get(w_obj) + try: + weakref.ref(w_obj) + except TypeError: + lifeline = None + else: + lifeline = state.lifeline_dict.get(w_obj) if lifeline is not None: refcnt = lifeline.pyo.c_ob_refcnt if refcnt > 0: @@ -137,6 +154,8 @@ state = cls.space.fromcache(RefcountState) state.non_heaptypes_w[:] = [] + cls.w_cleanup_references = cls.space.wrap(interp2app(cls.cleanup_references)) + def compile_module(self, name, **kwds): """ Build an extension module linked against the cpyext api library. @@ -256,13 +275,7 @@ def teardown_method(self, func): for name in self.imported_module_names: self.unimport_module(name) - state = self.space.fromcache(RefcountState) - for w_obj in state.non_heaptypes_w: - Py_DecRef(self.space, w_obj) - state.non_heaptypes_w[:] = [] - state.reset_borrowed_references() - from pypy.module.cpyext import cdatetime - cdatetime.datetimeAPI_dealloc(self.space) + self.cleanup_references(self.space) if self.check_and_print_leaks(): assert False, "Test leaks or loses object(s)." @@ -669,3 +682,19 @@ assert mod.get_names() == ('cell', 'module', 'property', 'staticmethod', 'builtin_function_or_method') + + def test_get_programname(self): + mod = self.import_extension('foo', [ + ('get_programname', 'METH_NOARGS', + ''' + char* name1 = Py_GetProgramName(); + char* name2 = Py_GetProgramName(); + if (name1 != name2) + Py_RETURN_FALSE; + return PyString_FromString(name1); + ''' + ), + ]) + p = mod.get_programname() + print p + assert 'py' in p Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_datetime.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_datetime.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_datetime.py Sat Dec 11 15:10:15 2010 @@ -92,9 +92,20 @@ PyDateTimeAPI->TimeType, PyDateTimeAPI->DeltaType); """), + ("clear_types", "METH_NOARGS", + """ + Py_DECREF(PyDateTimeAPI->DateType); + Py_DECREF(PyDateTimeAPI->DateTimeType); + Py_DECREF(PyDateTimeAPI->TimeType); + Py_DECREF(PyDateTimeAPI->DeltaType); + Py_RETURN_NONE; + """ + ) ]) import datetime assert module.get_types() == (datetime.date, datetime.datetime, datetime.time, datetime.timedelta) + module.clear_types() + self.cleanup_references() Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_intobject.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_intobject.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_intobject.py Sat Dec 11 15:10:15 2010 @@ -19,6 +19,10 @@ assert api.PyErr_Occurred() is space.w_TypeError api.PyErr_Clear() + assert api.PyInt_AsLong(None) == -1 + assert api.PyErr_Occurred() is space.w_TypeError + api.PyErr_Clear() + assert api.PyInt_AsUnsignedLong(space.wrap(sys.maxint)) == sys.maxint assert api.PyInt_AsUnsignedLong(space.wrap(-5)) == sys.maxint * 2 + 1 assert api.PyErr_Occurred() is space.w_ValueError Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_object.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_object.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_object.py Sat Dec 11 15:10:15 2010 @@ -4,7 +4,7 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.rpython.lltypesystem import rffi, lltype from pypy.module.cpyext.api import Py_LT, Py_LE, Py_NE, Py_EQ,\ - Py_GE, Py_GT + Py_GE, Py_GT, fopen, fclose, fwrite from pypy.tool.udir import udir class TestObject(BaseApiTest): @@ -188,10 +188,45 @@ rffi.free_charp(filename) rffi.free_charp(mode) + assert api.PyFile_Check(w_file) + assert api.PyFile_CheckExact(w_file) + assert not api.PyFile_Check(space.wrap("text")) + space.call_method(w_file, "write", space.wrap("text")) space.call_method(w_file, "close") assert (udir / "_test_file").read() == "text" + def test_file_getline(self, space, api): + filename = rffi.str2charp(str(udir / "_test_file")) + + mode = rffi.str2charp("w") + w_file = api.PyFile_FromString(filename, mode) + space.call_method(w_file, "write", + space.wrap("line1\nline2\nline3\nline4")) + space.call_method(w_file, "close") + + rffi.free_charp(mode) + mode = rffi.str2charp("r") + w_file = api.PyFile_FromString(filename, mode) + rffi.free_charp(filename) + rffi.free_charp(mode) + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "line1\n" + + w_line = api.PyFile_GetLine(w_file, 4) + assert space.str_w(w_line) == "line" + + w_line = api.PyFile_GetLine(w_file, 0) + assert space.str_w(w_line) == "2\n" + + # XXX We ought to raise an EOFError here, but don't + w_line = api.PyFile_GetLine(w_file, -1) + # assert api.PyErr_Occurred() is space.w_EOFError + assert space.str_w(w_line) == "line3\n" + + space.call_method(w_file, "close") + class AppTestObject(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_pyerrors.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_pyerrors.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_pyerrors.py Sat Dec 11 15:10:15 2010 @@ -129,6 +129,41 @@ ]) assert module.check_error() + + def test_normalize(self): + module = self.import_extension('foo', [ + ("check_error", "METH_NOARGS", + ''' + PyObject *type, *val, *tb; + PyErr_SetString(PyExc_TypeError, "message"); + + PyErr_Fetch(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (!PyString_Check(val)) + Py_RETURN_FALSE; + /* Normalize */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + /* Normalize again */ + PyErr_NormalizeException(&type, &val, &tb); + if (type != PyExc_TypeError) + Py_RETURN_FALSE; + if (val->ob_type != PyExc_TypeError) + Py_RETURN_FALSE; + + PyErr_Restore(type, val, tb); + PyErr_Clear(); + Py_RETURN_TRUE; + ''' + ), + ]) + assert module.check_error() + def test_SetFromErrno(self): import sys if sys.platform != 'win32': @@ -149,3 +184,26 @@ except OSError, e: assert e.errno == errno.EBADF assert e.strerror == os.strerror(errno.EBADF) + assert e.filename == None + + def test_SetFromErrnoWithFilename(self): + import sys + if sys.platform != 'win32': + skip("callbacks through ll2ctypes modify errno") + import errno, os + + module = self.import_extension('foo', [ + ("set_from_errno", "METH_NOARGS", + ''' + errno = EBADF; + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "blyf"); + return NULL; + '''), + ], + prologue="#include ") + try: + module.set_from_errno() + except OSError, e: + assert e.filename == "blyf" + assert e.errno == errno.EBADF + assert e.strerror == os.strerror(errno.EBADF) Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_sequence.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_sequence.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_sequence.py Sat Dec 11 15:10:15 2010 @@ -70,3 +70,11 @@ assert space.unwrap(space.next(w_iter)) == 2 exc = raises(OperationError, space.next, w_iter) assert exc.value.match(space, space.w_StopIteration) + + def test_contains(self, space, api): + w_t = space.wrap((1, 'ha')) + assert api.PySequence_Contains(w_t, space.wrap(u'ha')) + assert not api.PySequence_Contains(w_t, space.wrap(2)) + assert api.PySequence_Contains(space.w_None, space.wrap(2)) == -1 + assert api.PyErr_Occurred() + api.PyErr_Clear() Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_typeobject.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_typeobject.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/test/test_typeobject.py Sat Dec 11 15:10:15 2010 @@ -20,6 +20,7 @@ assert type(obj) is module.fooType print "type of obj has type", type(type(obj)) print "type of type of obj has type", type(type(type(obj))) + self.cleanup_references() def test_typeobject_method_descriptor(self): module = self.import_module(name='foo') @@ -38,6 +39,7 @@ print obj.foo assert obj.foo == 42 assert obj.int_member == obj.foo + self.cleanup_references() def test_typeobject_data_member(self): module = self.import_module(name='foo') @@ -54,6 +56,7 @@ raises(SystemError, "obj.broken_member = 42") assert module.fooType.broken_member.__doc__ is None assert module.fooType.object_member.__doc__ == "A Python object." + self.cleanup_references() def test_typeobject_object_member(self): module = self.import_module(name='foo') @@ -74,6 +77,7 @@ obj.set_foo = 32 assert obj.foo == 32 + self.cleanup_references() def test_typeobject_string_member(self): module = self.import_module(name='foo') @@ -91,6 +95,7 @@ assert obj.char_member == "a" raises(TypeError, "obj.char_member = 'spam'") raises(TypeError, "obj.char_member = 42") + self.cleanup_references() def test_staticmethod(self): module = self.import_module(name="foo") @@ -98,6 +103,7 @@ assert obj.foo == 42 obj2 = obj.create() assert obj2.foo == 42 + self.cleanup_references() def test_new(self): module = self.import_module(name='foo') @@ -118,7 +124,8 @@ return self assert fuu2(u"abc").baz().escape() raises(TypeError, module.fooType.object_member.__get__, 1) - + self.cleanup_references() + def test_init(self): module = self.import_module(name="foo") newobj = module.FuuType() @@ -137,6 +144,7 @@ newobj = Fuu2() assert newobj.get_val() == 42 assert newobj.foobar == 32 + self.cleanup_references() def test_metatype(self): module = self.import_module(name='foo') @@ -145,6 +153,7 @@ assert isinstance(x, type) assert isinstance(x, module.MetaType) x() + self.cleanup_references() def test_metaclass_compatible(self): # metaclasses should not conflict here @@ -153,7 +162,9 @@ assert type(module.fooType).__mro__ == (type, object) y = module.MetaType('other', (module.fooType,), {}) assert isinstance(y, module.MetaType) - y() + x = y() + del x, y + self.cleanup_references() def test_sre(self): module = self.import_module(name='_sre') @@ -172,17 +183,21 @@ assert "groupdict" in dir(m) re._cache.clear() re._cache_repl.clear() + del prog, m + self.cleanup_references() def test_init_error(self): module = self.import_module("foo") raises(ValueError, module.InitErrType) - + self.cleanup_references() + def test_cmps(self): module = self.import_module("comparisons") cmpr = module.CmpType() assert cmpr == 3 assert cmpr != 42 - + self.cleanup_references() + def test_hash(self): module = self.import_module("comparisons") cmpr = module.CmpType() @@ -191,6 +206,7 @@ d[cmpr] = 72 assert d[cmpr] == 72 assert d[3] == 72 + self.cleanup_references() def test_descriptor(self): module = self.import_module("foo") @@ -205,6 +221,7 @@ assert obj.y == (prop, 2) del obj.x assert obj.z == prop + self.cleanup_references() def test_tp_dict(self): foo = self.import_module("foo") @@ -226,6 +243,7 @@ ]) obj = foo.new() assert module.read_tp_dict(obj) == foo.fooType.copy + self.cleanup_references() class TestTypes(BaseApiTest): Modified: pypy/branch/jit-unroll-loops/pypy/module/cpyext/typeobject.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/cpyext/typeobject.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/cpyext/typeobject.py Sat Dec 11 15:10:15 2010 @@ -24,7 +24,7 @@ from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne from pypy.module.cpyext.typeobjectdefs import ( PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc, - PyNumberMethods) + PyNumberMethods, PySequenceMethods) from pypy.module.cpyext.slotdefs import ( slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) from pypy.interpreter.error import OperationError @@ -136,6 +136,8 @@ if not struct: if slot_names[0] == 'c_tp_as_number': STRUCT_TYPE = PyNumberMethods + elif slot_names[0] == 'c_tp_as_sequence': + STRUCT_TYPE = PySequenceMethods else: raise AssertionError( "Structure not allocated: %s" % (slot_names[0],)) @@ -392,6 +394,8 @@ lltype.free(obj_pto.c_tp_as_buffer, flavor='raw') if obj_pto.c_tp_as_number: lltype.free(obj_pto.c_tp_as_number, flavor='raw') + if obj_pto.c_tp_as_sequence: + lltype.free(obj_pto.c_tp_as_sequence, flavor='raw') Py_DecRef(space, base_pyo) rffi.free_charp(obj_pto.c_tp_name) PyObject_dealloc(space, obj) @@ -433,12 +437,6 @@ finish_type_1(space, pto) finish_type_2(space, pto, w_type) - if space.type(w_type).is_cpytype(): - # XXX Types with a C metatype are never freed, try to see why... - render_immortal(pto, w_type) - lltype.render_immortal(pto) - lltype.render_immortal(pto.c_tp_name) - pto.c_tp_basicsize = rffi.sizeof(typedescr.basestruct) if pto.c_tp_base: if pto.c_tp_base.c_tp_basicsize > pto.c_tp_basicsize: @@ -544,25 +542,12 @@ w_obj.ready() finish_type_2(space, py_type, w_obj) - render_immortal(py_type, w_obj) state = space.fromcache(RefcountState) state.non_heaptypes_w.append(w_obj) return w_obj -def render_immortal(py_type, w_obj): - lltype.render_immortal(py_type.c_tp_bases) - lltype.render_immortal(py_type.c_tp_mro) - - assert isinstance(w_obj, W_TypeObject) - if w_obj.is_cpytype(): - lltype.render_immortal(py_type.c_tp_dict) - else: - lltype.render_immortal(py_type.c_tp_name) - if not w_obj.is_cpytype() and w_obj.is_heaptype(): - lltype.render_immortal(py_type) - def finish_type_1(space, pto): """ Sets up tp_bases, necessary before creating the interpreter type. @@ -599,7 +584,8 @@ if w_obj.is_cpytype(): Py_DecRef(space, pto.c_tp_dict) - pto.c_tp_dict = make_ref(space, w_obj.getdict()) + w_dict = space.newdict(from_strdict_shared=w_obj.dict_w) + pto.c_tp_dict = make_ref(space, w_dict) @cpython_api([PyTypeObjectPtr, PyTypeObjectPtr], rffi.INT_real, error=CANNOT_FAIL) def PyType_IsSubtype(space, a, b): Modified: pypy/branch/jit-unroll-loops/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/exceptions/interp_exceptions.py Sat Dec 11 15:10:15 2010 @@ -452,6 +452,7 @@ self.w_text = space.w_None self.w_msg = space.wrap('') self.w_print_file_and_line = space.w_None # what's that? + self.w_lastlineno = space.w_None # this is a pypy extension W_BaseException.__init__(self, space) def descr_init(self, space, args_w): @@ -459,11 +460,16 @@ if len(args_w) > 0: self.w_msg = args_w[0] if len(args_w) == 2: - values_w = space.fixedview(args_w[1], 4) - self.w_filename = values_w[0] - self.w_lineno = values_w[1] - self.w_offset = values_w[2] - self.w_text = values_w[3] + values_w = space.fixedview(args_w[1]) + if len(values_w) > 0: self.w_filename = values_w[0] + if len(values_w) > 1: self.w_lineno = values_w[1] + if len(values_w) > 2: self.w_offset = values_w[2] + if len(values_w) > 3: self.w_text = values_w[3] + if len(values_w) > 4: + self.w_lastlineno = values_w[4] # PyPy extension + # kill the extra items from args_w to prevent undesired effects + args_w = args_w[:] + args_w[1] = space.newtuple(values_w[:4]) W_BaseException.descr_init(self, space, args_w) descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] @@ -472,29 +478,52 @@ if type(self.msg) is not str: return str(self.msg) + lineno = None buffer = self.msg have_filename = type(self.filename) is str - have_lineno = type(self.lineno) is int + if type(self.lineno) is int: + if (type(self.lastlineno) is int and + self.lastlineno > self.lineno): + lineno = 'lines %d-%d' % (self.lineno, self.lastlineno) + else: + lineno = 'line %d' % (self.lineno,) if have_filename: import os fname = os.path.basename(self.filename or "???") - if have_lineno: - buffer = "%s (%s, line %ld)" % (self.msg, fname, self.lineno) + if lineno: + buffer = "%s (%s, %s)" % (self.msg, fname, lineno) else: buffer ="%s (%s)" % (self.msg, fname) - elif have_lineno: - buffer = "%s (line %ld)" % (self.msg, self.lineno) + elif lineno: + buffer = "%s (%s)" % (self.msg, lineno) return buffer """) descr_str.unwrap_spec = ['self', ObjSpace] + def descr_repr(self, space): + if (len(self.args_w) == 2 + and not space.is_w(self.w_lastlineno, space.w_None) + and space.int_w(space.len(self.args_w[1])) == 4): + # fake a 5-element tuple in the repr, suitable for calling + # __init__ again + values_w = space.fixedview(self.args_w[1]) + w_tuple = space.newtuple(values_w + [self.w_lastlineno]) + args_w = [self.args_w[0], w_tuple] + args_repr = space.str_w(space.repr(space.newtuple(args_w))) + clsname = self.getclass(space).getname(space, '?') + return space.wrap(clsname + args_repr) + else: + return W_StandardError.descr_repr(self, space) + descr_repr.unwrap_spec = ['self', ObjSpace] + W_SyntaxError.typedef = TypeDef( 'SyntaxError', W_StandardError.typedef, __new__ = _new(W_SyntaxError), __init__ = interp2app(W_SyntaxError.descr_init), __str__ = interp2app(W_SyntaxError.descr_str), + __repr__ = interp2app(W_SyntaxError.descr_repr), __doc__ = W_SyntaxError.__doc__, __module__ = 'exceptions', msg = readwrite_attrproperty_w('w_msg', W_SyntaxError), @@ -504,6 +533,7 @@ text = readwrite_attrproperty_w('w_text', W_SyntaxError), print_file_and_line = readwrite_attrproperty_w('w_print_file_and_line', W_SyntaxError), + lastlineno = readwrite_attrproperty_w('w_lastlineno', W_SyntaxError), ) W_FutureWarning = _new_exception('FutureWarning', W_Warning, Modified: pypy/branch/jit-unroll-loops/pypy/module/fcntl/test/test_fcntl.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/fcntl/test/test_fcntl.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/fcntl/test/test_fcntl.py Sat Dec 11 15:10:15 2010 @@ -63,7 +63,7 @@ if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3', 'Darwin1.2', 'darwin', 'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5', - 'freebsd6', 'freebsd7', + 'freebsd6', 'freebsd7', 'freebsd8', 'freebsd9', 'bsdos2', 'bsdos3', 'bsdos4', 'openbsd', 'openbsd2', 'openbsd3'): if struct.calcsize('l') == 8: @@ -159,7 +159,7 @@ if "linux" in sys.platform: TIOCGPGRP = 0x540f - elif "darwin" in sys.platform or "freebsd6" == sys.platform: + elif "darwin" in sys.platform or "freebsd" in sys.platform: TIOCGPGRP = 0x40047477 else: skip("don't know how to test ioctl() on this platform") Modified: pypy/branch/jit-unroll-loops/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/imp/importing.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/imp/importing.py Sat Dec 11 15:10:15 2010 @@ -76,7 +76,7 @@ return SEARCH_ERROR, None, None -if sys.platform in ['linux2', 'freebsd']: +if sys.platform == 'linux2' or 'freebsd' in sys.platform: def case_ok(filename): return True else: @@ -529,7 +529,7 @@ space.sys.setmodule(w_module) raise finally: - space.reloading_modules.clear() + del space.reloading_modules[modulename] # __________________________________________________________________ Modified: pypy/branch/jit-unroll-loops/pypy/module/itertools/interp_itertools.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/itertools/interp_itertools.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/itertools/interp_itertools.py Sat Dec 11 15:10:15 2010 @@ -391,7 +391,8 @@ self.iterators_w = iterators_w self.current_iterator = 0 self.num_iterators = len(iterators_w) - self.started = False + if self.num_iterators > 0: + self.w_it = iterators_w[0] def iter_w(self): return self.space.wrap(self) @@ -399,26 +400,23 @@ def next_w(self): if self.current_iterator >= self.num_iterators: raise OperationError(self.space.w_StopIteration, self.space.w_None) - if not self.started: - self.current_iterator = 0 - self.w_it = self.iterators_w[self.current_iterator] - self.started = True + try: + return self.space.next(self.w_it) + except OperationError, e: + return self._handle_error(e) + def _handle_error(self, e): while True: + if not e.match(self.space, self.space.w_StopIteration): + raise e + self.current_iterator += 1 + if self.current_iterator >= self.num_iterators: + raise e + self.w_it = self.iterators_w[self.current_iterator] try: - w_obj = self.space.next(self.w_it) + return self.space.next(self.w_it) except OperationError, e: - if e.match(self.space, self.space.w_StopIteration): - self.current_iterator += 1 - if self.current_iterator >= self.num_iterators: - raise OperationError(self.space.w_StopIteration, self.space.w_None) - else: - self.w_it = self.iterators_w[self.current_iterator] - else: - raise - else: - break - return w_obj + pass # loop back to the start of _handle_error(e) def W_Chain___new__(space, w_subtype, args_w): return space.wrap(W_Chain(space, args_w)) @@ -446,8 +444,10 @@ def __init__(self, space, w_fun, args_w): self.space = space - self.identity_fun = (self.space.is_w(w_fun, space.w_None)) - self.w_fun = w_fun + if self.space.is_w(w_fun, space.w_None): + self.w_fun = None + else: + self.w_fun = w_fun iterators_w = [] i = 0 @@ -470,12 +470,26 @@ return self.space.wrap(self) def next_w(self): - w_objects = self.space.newtuple([self.space.next(w_it) for w_it in self.iterators_w]) - if self.identity_fun: + # common case: 1 or 2 arguments + iterators_w = self.iterators_w + length = len(iterators_w) + if length == 1: + objects = [self.space.next(iterators_w[0])] + elif length == 2: + objects = [self.space.next(iterators_w[0]), + self.space.next(iterators_w[1])] + else: + objects = self._get_objects() + w_objects = self.space.newtuple(objects) + if self.w_fun is None: return w_objects else: return self.space.call(self.w_fun, w_objects) + def _get_objects(self): + # the loop is out of the way of the JIT + return [self.space.next(w_elem) for w_elem in self.iterators_w] + def W_IMap___new__(space, w_subtype, w_fun, args_w): if len(args_w) == 0: @@ -769,15 +783,7 @@ raise OperationError(self.space.w_StopIteration, self.space.w_None) if not self.new_group: - # Consume unwanted input until we reach the next group - try: - while True: - self.group_next(self.index) - - except StopIteration: - pass - if self.exhausted: - raise OperationError(self.space.w_StopIteration, self.space.w_None) + self._consume_unwanted_input() if not self.started: self.started = True @@ -799,6 +805,16 @@ w_iterator = self.space.wrap(W_GroupByIterator(self.space, self.index, self)) return self.space.newtuple([self.w_key, w_iterator]) + def _consume_unwanted_input(self): + # Consume unwanted input until we reach the next group + try: + while True: + self.group_next(self.index) + except StopIteration: + pass + if self.exhausted: + raise OperationError(self.space.w_StopIteration, self.space.w_None) + def group_next(self, group_index): if group_index < self.index: raise StopIteration Modified: pypy/branch/jit-unroll-loops/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/posix/__init__.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/posix/__init__.py Sat Dec 11 15:10:15 2010 @@ -77,6 +77,8 @@ interpleveldefs['fsync'] = 'interp_posix.fsync' if hasattr(os, 'fdatasync'): interpleveldefs['fdatasync'] = 'interp_posix.fdatasync' + if hasattr(os, 'fchdir'): + interpleveldefs['fchdir'] = 'interp_posix.fchdir' if hasattr(os, 'putenv'): interpleveldefs['putenv'] = 'interp_posix.putenv' if hasattr(posix, 'unsetenv'): # note: emulated in os @@ -96,6 +98,8 @@ interpleveldefs['fork'] = 'interp_posix.fork' if hasattr(os, 'openpty'): interpleveldefs['openpty'] = 'interp_posix.openpty' + if hasattr(os, 'forkpty'): + interpleveldefs['forkpty'] = 'interp_posix.forkpty' if hasattr(os, 'waitpid'): interpleveldefs['waitpid'] = 'interp_posix.waitpid' if hasattr(os, 'execv'): @@ -109,6 +113,8 @@ interpleveldefs['sysconf_names'] = 'space.wrap(os.sysconf_names)' if hasattr(os, 'ttyname'): interpleveldefs['ttyname'] = 'interp_posix.ttyname' + if hasattr(os, 'getloadavg'): + interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', Modified: pypy/branch/jit-unroll-loops/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/posix/interp_posix.py Sat Dec 11 15:10:15 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 +from pypy.interpreter.error import operationerrfmt from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -151,19 +152,29 @@ raise wrap_oserror(space, e) ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] -def fsync(space, fd): +def fsync(space, w_fd): + fd = space.c_filedescriptor_w(w_fd) try: os.fsync(fd) except OSError, e: raise wrap_oserror(space, e) -fsync.unwrap_spec = [ObjSpace, "c_int"] +fsync.unwrap_spec = [ObjSpace, W_Root] -def fdatasync(space, fd): +def fdatasync(space, w_fd): + fd = space.c_filedescriptor_w(w_fd) try: os.fdatasync(fd) except OSError, e: raise wrap_oserror(space, e) -fdatasync.unwrap_spec = [ObjSpace, "c_int"] +fdatasync.unwrap_spec = [ObjSpace, W_Root] + +def fchdir(space, w_fd): + fd = space.c_filedescriptor_w(w_fd) + try: + os.fchdir(fd) + except OSError, e: + raise wrap_oserror(space, e) +fchdir.unwrap_spec = [ObjSpace, W_Root] # ____________________________________________________________ @@ -608,6 +619,14 @@ raise wrap_oserror(space, e) return space.newtuple([space.wrap(master_fd), space.wrap(slave_fd)]) +def forkpty(space): + try: + pid, master_fd = os.forkpty() + except OSError, e: + raise wrap_oserror(space, e) + return space.newtuple([space.wrap(pid), + space.wrap(master_fd)]) + def waitpid(space, pid, options): """ waitpid(pid, options) -> (pid, status) @@ -955,6 +974,17 @@ return space.w_None chown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] +def getloadavg(space): + try: + load = os.getloadavg() + except OSError, e: + raise OperationError(space.w_OSError, + space.wrap("Load averages are unobtainable")) + return space.newtuple([space.wrap(load[0]), + space.wrap(load[1]), + space.wrap(load[2])]) +getloadavg.unwrap_spec = [ObjSpace] + if _WIN: from pypy.rlib import rwin32 Modified: pypy/branch/jit-unroll-loops/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/posix/test/test_posix2.py Sat Dec 11 15:10:15 2010 @@ -331,6 +331,22 @@ data = os.read(master_fd, 100) assert data.startswith('x') + if hasattr(__import__(os.name), "forkpty"): + def test_forkpty(self): + import sys + os = self.posix + childpid, master_fd = os.forkpty() + assert isinstance(childpid, int) + assert isinstance(master_fd, int) + if childpid == 0: + data = os.read(0, 100) + if data.startswith('abc'): + os._exit(42) + else: + os._exit(43) + os.write(master_fd, 'abc\n') + _, status = os.waitpid(childpid, 0) + assert status >> 8 == 42 if hasattr(__import__(os.name), "execv"): def test_execv(self): @@ -505,6 +521,14 @@ assert os.WIFEXITED(status) assert os.WEXITSTATUS(status) == exit_status + if hasattr(os, 'getloadavg'): + def test_os_getloadavg(self): + os = self.posix + l0, l1, l2 = os.getloadavg() + assert type(l0) is float and l0 >= 0.0 + assert type(l1) is float and l0 >= 0.0 + assert type(l2) is float and l0 >= 0.0 + if hasattr(os, 'fsync'): def test_fsync(self): os = self.posix @@ -512,30 +536,42 @@ try: fd = f.fileno() os.fsync(fd) - finally: + os.fsync(long(fd)) + os.fsync(f) # <- should also work with a file, or anything + finally: # with a fileno() method f.close() - try: - os.fsync(fd) - except OSError: - pass - else: - raise AssertionError("os.fsync didn't raise") + raises(OSError, os.fsync, fd) + raises(ValueError, os.fsync, -1) if hasattr(os, 'fdatasync'): def test_fdatasync(self): os = self.posix - f = open(self.path2) + f = open(self.path2, "w") try: fd = f.fileno() os.fdatasync(fd) finally: f.close() + raises(OSError, os.fdatasync, fd) + raises(ValueError, os.fdatasync, -1) + + if hasattr(os, 'fchdir'): + def test_fchdir(self): + os = self.posix + localdir = os.getcwd() try: - os.fdatasync(fd) - except OSError: - pass - else: - raise AssertionError("os.fdatasync didn't raise") + os.mkdir(self.path2 + 'dir') + fd = os.open(self.path2 + 'dir', os.O_RDONLY) + try: + os.fchdir(fd) + mypath = os.getcwd() + finally: + os.close(fd) + assert mypath.endswith('test_posix2-dir') + raises(OSError, os.fchdir, fd) + raises(ValueError, os.fchdir, -1) + finally: + os.chdir(localdir) def test_largefile(self): os = self.posix Modified: pypy/branch/jit-unroll-loops/pypy/module/pyexpat/interp_pyexpat.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/pyexpat/interp_pyexpat.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/pyexpat/interp_pyexpat.py Sat Dec 11 15:10:15 2010 @@ -329,7 +329,7 @@ if self.returns_unicode: return space.call_function( space.getattr(space.wrap(s), space.wrap("decode")), - space.wrap(self.encoding), + space.wrap("utf-8"), space.wrap("strict")) else: return space.wrap(s) Modified: pypy/branch/jit-unroll-loops/pypy/module/pyexpat/test/test_parser.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/pyexpat/test/test_parser.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/pyexpat/test/test_parser.py Sat Dec 11 15:10:15 2010 @@ -15,3 +15,17 @@ assert res == 1 raises(pyexpat.ExpatError, p.Parse, "3") + + def test_encoding(self): + import pyexpat + for encoding_arg in (None, 'utf-8', 'iso-8859-1'): + for namespace_arg in (None, '{'): + print encoding_arg, namespace_arg + p = pyexpat.ParserCreate(encoding_arg, namespace_arg) + data = [] + p.CharacterDataHandler = lambda s: data.append(s) + encoding = encoding_arg is None and 'utf-8' or encoding_arg + + res = p.Parse(u"\u00f6".encode(encoding), isfinal=True) + assert res == 1 + assert data == [u"\u00f6"] Modified: pypy/branch/jit-unroll-loops/pypy/module/pypyjit/__init__.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/pypyjit/__init__.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/pypyjit/__init__.py Sat Dec 11 15:10:15 2010 @@ -15,6 +15,5 @@ # add the 'defaults' attribute from pypy.rlib.jit import PARAMETERS space = self.space - # XXX this is not really the default compiled into a pypy-c-jit XXX w_obj = space.wrap(PARAMETERS) space.setattr(space.wrap(self), space.wrap('defaults'), w_obj) Modified: pypy/branch/jit-unroll-loops/pypy/module/pypyjit/policy.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/pypyjit/policy.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/pypyjit/policy.py Sat Dec 11 15:10:15 2010 @@ -12,19 +12,13 @@ if '.' in modname: modname, _ = modname.split('.', 1) if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions', - 'imp', 'sys', 'array', '_ffi']: + 'imp', 'sys', 'array', '_ffi', 'itertools', 'operator']: return True return False def look_inside_function(self, func): - # this function should never actually return True directly - # but instead call the base implementation mod = func.__module__ or '?' - if mod.startswith('pypy.objspace.'): - # gc_id operation - if func.__name__ == 'id__ANY': - return False if mod == 'pypy.rlib.rbigint' or mod == 'pypy.rlib.rlocale': return False if '_geninterp_' in func.func_globals: # skip all geninterped stuff Modified: pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_policy.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_policy.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_policy.py Sat Dec 11 15:10:15 2010 @@ -4,7 +4,7 @@ def test_id_any(): from pypy.objspace.std.default import id__ANY - assert not pypypolicy.look_inside_function(id__ANY) + assert pypypolicy.look_inside_function(id__ANY) def test_bigint(): from pypy.rlib.rbigint import rbigint Modified: pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_pypy_c.py Sat Dec 11 15:10:15 2010 @@ -688,6 +688,32 @@ ''', 3000, ([0], 2000*3)) assert len(self.loops) == 1 + def test_getattr_with_dynamic_attribute(self): + self.run_source(''' + class A(object): + pass + + l = ["x", "y"] + + def main(arg): + sum = 0 + a = A() + a.a1 = 0 + a.a2 = 0 + a.a3 = 0 + a.a4 = 0 + a.a5 = 0 # workaround, because the first five attributes need a promotion + a.x = 1 + a.y = 2 + i = 0 + while i < 2000: + name = l[i % 2] + sum += getattr(a, name) + i += 1 + return sum + ''', 3000, ([0], 3000)) + assert len(self.loops) == 1 + def test_blockstack_virtualizable(self): self.run_source(''' from pypyjit import residual_call @@ -731,11 +757,9 @@ i = t2[3] del t2 return i - ''', 100, ([], 100)) + ''', 40, ([], 100)) bytecode, = self.get_by_bytecode('BINARY_SUBSCR') - assert len(bytecode.get_opnames('new_array')) == 1 - # XXX I would like here to say that it's 0, but unfortunately - # call that can raise is not exchanged into getarrayitem_gc + assert len(bytecode.get_opnames('new_array')) == 0 def test_overflow_checking(self): startvalue = sys.maxint - 2147483647 @@ -919,11 +943,14 @@ def test_array_sum(self): for tc, maxops in zip('bhilBHILfd', (38,) * 6 + (40, 40, 41, 38)): res = 19352859 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array @@ -971,11 +998,14 @@ print '='*65 print '='*20, 'running test for tc=%r' % (tc,), '='*20 res = 73574560 - if tc in 'IL': + if tc == 'L': res = long(res) elif tc in 'fd': res = float(res) - + elif tc == 'I' and sys.maxint == 2147483647: + res = long(res) + # note: in CPython we always get longs here, even on 64-bits + self.run_source(''' from array import array Modified: pypy/branch/jit-unroll-loops/pypy/module/select/interp_select.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/select/interp_select.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/select/interp_select.py Sat Dec 11 15:10:15 2010 @@ -12,37 +12,17 @@ unregistering file descriptors, and then polling them for I/O events.""" return Poll() -def as_fd_w(space, w_fd): - if not space.is_true(space.isinstance(w_fd, space.w_int)): - try: - w_fileno = space.getattr(w_fd, space.wrap('fileno')) - except OperationError, e: - if e.match(space, space.w_AttributeError): - raise OperationError(space.w_TypeError, - space.wrap("argument must be an int, or have a fileno() method.")) - raise - w_fd = space.call_function(w_fileno) - if not space.is_true(space.isinstance(w_fd, space.w_int)): - raise OperationError(space.w_TypeError, - space.wrap('filneo() return a non-integer')) - - fd = space.int_w(w_fd) - if fd < 0: - raise operationerrfmt(space.w_ValueError, - "file descriptor cannot be a negative integer (%d)", fd) - return fd - class Poll(Wrappable): def __init__(self): self.fddict = {} def register(self, space, w_fd, events=defaultevents): - fd = as_fd_w(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) self.fddict[fd] = events register.unwrap_spec = ['self', ObjSpace, W_Root, int] def unregister(self, space, w_fd): - fd = as_fd_w(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) try: del self.fddict[fd] except KeyError: @@ -113,9 +93,9 @@ iwtd_w = space.listview(w_iwtd) owtd_w = space.listview(w_owtd) ewtd_w = space.listview(w_ewtd) - iwtd = [as_fd_w(space, w_f) for w_f in iwtd_w] - owtd = [as_fd_w(space, w_f) for w_f in owtd_w] - ewtd = [as_fd_w(space, w_f) for w_f in ewtd_w] + iwtd = [space.c_filedescriptor_w(w_f) for w_f in iwtd_w] + owtd = [space.c_filedescriptor_w(w_f) for w_f in owtd_w] + ewtd = [space.c_filedescriptor_w(w_f) for w_f in ewtd_w] iwtd_d = {} owtd_d = {} ewtd_d = {} Modified: pypy/branch/jit-unroll-loops/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/sys/__init__.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/sys/__init__.py Sat Dec 11 15:10:15 2010 @@ -60,8 +60,6 @@ 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', 'hexversion' : 'version.get_hexversion(space)', - 'ps1' : 'space.wrap(">>>> ")', - 'ps2' : 'space.wrap(".... ")', 'displayhook' : 'hook.displayhook', '__displayhook__' : 'hook.__displayhook__', Modified: pypy/branch/jit-unroll-loops/pypy/module/sys/state.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/sys/state.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/sys/state.py Sat Dec 11 15:10:15 2010 @@ -33,6 +33,8 @@ raise OSError(errno.ENOTDIR, path) +platform = sys.platform + def getinitialpath(prefix): from pypy.module.sys.version import CPYTHON_VERSION dirname = '%d.%d.%d' % (CPYTHON_VERSION[0], @@ -51,6 +53,15 @@ importlist.append(lib_pypy) importlist.append(python_std_lib_modified) importlist.append(python_std_lib) + # + # List here the extra platform-specific paths. + if platform != 'win32': + importlist.append(os.path.join(python_std_lib, 'plat-'+platform)) + if platform == 'darwin': + platmac = os.path.join(python_std_lib, 'plat-mac') + importlist.append(platmac) + importlist.append(os.path.join(platmac, 'lib-scriptpackages')) + # return importlist def pypy_initial_path(space, srcdir): Modified: pypy/branch/jit-unroll-loops/pypy/module/sys/test/test_initialpath.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/sys/test/test_initialpath.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/sys/test/test_initialpath.py Sat Dec 11 15:10:15 2010 @@ -15,4 +15,5 @@ def test_stdlib_in_prefix(tmpdir): dirs = build_hierarchy(tmpdir) path = getinitialpath(str(tmpdir)) - assert path == map(str, dirs) + # we get at least 'dirs', and maybe more (e.g. plat-linux2) + assert path[:len(dirs)] == map(str, dirs) Modified: pypy/branch/jit-unroll-loops/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/sys/vm.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/sys/vm.py Sat Dec 11 15:10:15 2010 @@ -41,27 +41,21 @@ f = ec.getnextframe_nohidden(f) return space.wrap(f) -# directly from the C code in ceval.c, might be moved somewhere else. - def setrecursionlimit(space, w_new_limit): - """Set the maximum depth of the Python interpreter stack to n. This -limit prevents infinite recursion from causing an overflow of the C -stack and crashing Python. The highest possible limit is platform -dependent.""" + """DEPRECATED on PyPy. Will issue warning and not work + """ new_limit = space.int_w(w_new_limit) if new_limit <= 0: raise OperationError(space.w_ValueError, space.wrap("recursion limit must be positive")) # global recursion_limit # we need to do it without writing globals. + space.warn('setrecursionlimit deprecated', space.w_DeprecationWarning) space.sys.recursionlimit = new_limit def getrecursionlimit(space): - """Return the current value of the recursion limit, the maximum depth - of the Python interpreter stack. This limit prevents infinite - recursion from causing an overflow of the C stack and crashing Python. + """DEPRECATED on PyPy. Will issue warning and not work """ - return space.wrap(space.sys.recursionlimit) def setcheckinterval(space, interval): Modified: pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py Sat Dec 11 15:10:15 2010 @@ -82,8 +82,11 @@ def test_cast_functype(self): # make sure we can cast function type my_sqrt = lib.my_sqrt + saved_objects = my_sqrt._objects.copy() sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) assert sqrt(4.0) == 2.0 assert not cast(0, CFUNCTYPE(c_int)) - - + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) Modified: pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_guess_argtypes.py Sat Dec 11 15:10:15 2010 @@ -11,21 +11,23 @@ py.test.skip("pypy white-box test") from _ctypes.function import CFuncPtr - guess = CFuncPtr._guess_argtypes + def guess(value): + cobj = CFuncPtr._conv_param(None, value, 0) + return type(cobj) - assert guess([13]) == [c_int] - assert guess([0]) == [c_int] - assert guess(['xca']) == [c_char_p] - assert guess([None]) == [c_void_p] - assert guess([c_int(3)]) == [c_int] - assert guess([u'xca']) == [c_wchar_p] + assert guess(13) == c_int + assert guess(0) == c_int + assert guess('xca') == c_char_p + assert guess(None) == c_void_p + assert guess(c_int(3)) == c_int + assert guess(u'xca') == c_wchar_p class Stuff: pass s = Stuff() s._as_parameter_ = None - assert guess([s]) == [c_void_p] + assert guess(s) == c_void_p def test_guess_unicode(): if not hasattr(sys, 'pypy_translation_info') and sys.platform != 'win32': Modified: pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/test_lib_pypy/ctypes_tests/test_keepalive.py Sat Dec 11 15:10:15 2010 @@ -99,7 +99,7 @@ def test_primitive(self): if not hasattr(sys, 'pypy_translation_info'): py.test.skip("pypy white-box test") - assert c_char_p("abc")._objects['0']._buffer[0] == "a" + assert c_char_p("abc")._objects._buffer[0] == "a" assert c_int(3)._objects is None def test_pointer_to_pointer(self): @@ -123,7 +123,7 @@ pass cf = CFUNCTYPE(c_int, c_int)(f) p1 = cast(cf, c_void_p) - assert p1._objects == {'1': cf, '0': {'0': cf}} + assert p1._objects == {id(cf): cf, '0': cf} def test_array_of_struct_with_pointer(self): class S(Structure): @@ -221,7 +221,7 @@ import gc; gc.collect() print 'x =', repr(x) assert x.value == 'hellohello' - assert x._objects.keys() == ['0'] + assert x._objects == 'hellohello' # class datum(Structure): _fields_ = [ Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/complexobject.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/complexobject.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/complexobject.py Sat Dec 11 15:10:15 2010 @@ -23,90 +23,92 @@ """ representation for debugging purposes """ return "" % (w_self.realval, w_self.imagval) -registerimplementation(W_ComplexObject) - -c_1 = (1.0, 0.0) + def sub(self, other): + return W_ComplexObject(self.realval - other.realval, + self.imagval - other.imagval) + + def mul(self, other): + r = self.realval * other.realval - self.imagval * other.imagval + i = self.realval * other.imagval + self.imagval * other.realval + return W_ComplexObject(r, i) + + def div(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 < 0: + abs_r2 = - r2 + else: + abs_r2 = r2 + if i2 < 0: + abs_i2 = - i2 + else: + abs_i2 = i2 + if abs_r2 >= abs_i2: + if abs_r2 == 0.0: + raise ZeroDivisionError + else: + ratio = i2 / r2 + denom = r2 + i2 * ratio + rr = (r1 + i1 * ratio) / denom + ir = (i1 - r1 * ratio) / denom + else: + ratio = r2 / i2 + denom = r2 * ratio + i2 + assert i2 != 0.0 + rr = (r1 * ratio + i1) / denom + ir = (i1 * ratio - r1) / denom + return W_ComplexObject(rr,ir) + + def divmod(self, other): + w_div = self.div(other) + div = math.floor(w_div.realval) + w_mod = self.sub( + W_ComplexObject(other.realval * div, other.imagval * div)) + return (W_ComplexObject(div, 0), w_mod) + + def pow(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 == 0.0 and i2 == 0.0: + rr, ir = 1, 0 + elif r1 == 0.0 and i1 == 0.0: + if i2 != 0.0 or r2 < 0.0: + raise ZeroDivisionError + rr, ir = (0.0, 0.0) + else: + vabs = math.hypot(r1,i1) + len = math.pow(vabs,r2) + at = math.atan2(i1,r1) + phase = at * r2 + if i2 != 0.0: + len /= math.exp(at * i2) + phase += i2 * math.log(vabs) + rr = len * math.cos(phase) + ir = len * math.sin(phase) + return W_ComplexObject(rr, ir) + + def pow_int(self, n): + if n > 100 or n < -100: + return self.pow(W_ComplexObject(1.0 * n, 0.0)) + elif n > 0: + return self.pow_positive_int(n) + else: + return w_one.div(self.pow_positive_int(-n)) -def _sum(c1, c2): - return (c1[0]+c2[0],c1[1]+c2[1]) + def pow_positive_int(self, n): + mask = 1 + w_result = w_one + while mask > 0 and n >= mask: + if n & mask: + w_result = w_result.mul(self) + mask <<= 1 + self = self.mul(self) -def _diff(c1, c2): - return (c1[0]-c2[0],c1[1]-c2[1]) + return w_result -def _prod(c1, c2): - r = c1[0]*c2[0] - c1[1]*c2[1] - i = c1[0]*c2[1] + c1[1]*c2[0] - return (r,i) - -def _quot(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 < 0: - abs_r2 = - r2 - else: - abs_r2 = r2 - if i2 < 0: - abs_i2 = - i2 - else: - abs_i2 = i2 - if abs_r2 >= abs_i2: - if abs_r2 == 0.0: - raise ZeroDivisionError - else: - ratio = i2 / r2 - denom = r2 + i2 * ratio - rr = (r1 + i1 * ratio) / denom - ir = (i1 - r1 * ratio) / denom - else: - ratio = r2 / i2 - denom = r2 * ratio + i2 - assert i2 != 0.0 - rr = (r1 * ratio + i1) / denom - ir = (i1 * ratio - r1) / denom - return (rr,ir) - -def _pow(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 == 0.0 and i2 == 0.0: - rr, ir = c_1 - elif r1 == 0.0 and i1 == 0.0: - if i2 != 0.0 or r2 < 0.0: - raise ZeroDivisionError - rr, ir = (0.0, 0.0) - else: - vabs = math.hypot(r1,i1) - len = math.pow(vabs,r2) - at = math.atan2(i1,r1) - phase = at * r2 - if i2 != 0.0: - len /= math.exp(at * i2) - phase += i2 * math.log(vabs) - rr = len * math.cos(phase) - ir = len * math.sin(phase) - return (rr, ir) - -def _powu(c,n): - mask = 1; - rr, ir = c_1 - rp = c[0] - ip = c[1] - while mask > 0 and n >= mask: - if n & mask: - rr, ir = _prod((rr, ir), (rp, ip)) - mask <<= 1 - rp, ip = _prod((rp, ip), (rp, ip)) - - return (rr, ir) - -def _powi(c,n): - if n > 100 or n < -100: - return _pow(c,(1.0 * n, 0.0)) - elif n > 0: - return _powu(c, n) - else: - return _quot(c_1, _powu(c, -n)) +registerimplementation(W_ComplexObject) +w_one = W_ComplexObject(1, 0) def delegate_Bool2Complex(space, w_bool): @@ -126,38 +128,25 @@ return W_ComplexObject(w_float.floatval, 0.0) def hash__Complex(space, w_value): - #this is straight out of CPython complex implementation - hashreal = _hash_float(space, w_value.realval) - if hashreal == -1: - return space.newint(-1) hashimg = _hash_float(space, w_value.imagval) - if hashimg == -1: - return space.newint(-1) combined = hashreal + 1000003 * hashimg - if (combined == -1): - combined = -2 return space.newint(combined) -def _w2t(space, w_complex): - "convert an interplevel complex object to a tuple representation" - return w_complex.realval, w_complex.imagval - -def _t2w(space, c): - return W_ComplexObject(c[0], c[1]) - def add__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _sum(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval + w_complex2.realval, + w_complex1.imagval + w_complex2.imagval) def sub__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _diff(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval - w_complex2.realval, + w_complex1.imagval - w_complex2.imagval) def mul__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _prod(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.mul(w_complex2) def div__Complex_Complex(space, w_complex1, w_complex2): try: - return _t2w(space, _quot(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.div(w_complex2) except ZeroDivisionError, e: raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) @@ -165,49 +154,38 @@ def mod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[1] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex remainder")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - - return _t2w(space, mod) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) def divmod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + div, mod = w_complex1.divmod(w_complex2) except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex divmod()")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - w_div = _t2w(space, div) - w_mod = _t2w(space, mod) - return space.newtuple([w_div, w_mod]) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) + return space.newtuple([div, mod]) def floordiv__Complex_Complex(space, w_complex1, w_complex2): + # don't care about the slight slowdown you get from using divmod try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[0] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex floordiv()")) - div = (math.floor(div[0]), 0.0) - return _t2w(space, div) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) -def pow__Complex_Complex_ANY(space, w_complex1, w_complex2, thirdArg): +def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) + int_exponent = int(w_exponent.realval) try: - v = _w2t(space, w_complex1) - exponent = _w2t(space, w_complex2) - int_exponent = int(exponent[0]) - if exponent[1] == 0.0 and exponent[0] == int_exponent: - p = _powi(v, int_exponent) + if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: + w_p = w_complex.pow_int(int_exponent) else: - p = _pow(v, exponent) + w_p = w_complex.pow(w_exponent) except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("0.0 to a negative or complex power")) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("complex exponentiation")) - return _t2w(space, p) + return w_p def neg__Complex(space, w_complex): return W_ComplexObject(-w_complex.realval, -w_complex.imagval) Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/complextype.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/complextype.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/complextype.py Sat Dec 11 15:10:15 2010 @@ -1,7 +1,7 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.strutil import interp_string_to_float, ParseStringError +from pypy.objspace.std.strutil import string_to_float, ParseStringError from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod @@ -131,8 +131,8 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) try: - realval = interp_string_to_float(space, realstr) - imagval = interp_string_to_float(space, imagstr) + realval = string_to_float(realstr) + imagval = string_to_float(imagstr) except ParseStringError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) else: Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/floattype.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/floattype.py Sat Dec 11 15:10:15 2010 @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.objspace.std.stdtypedef import StdTypeDef from pypy.objspace.std.strutil import ParseStringError -from pypy.objspace.std.strutil import interp_string_to_float +from pypy.objspace.std.strutil import string_to_float def descr__new__(space, w_floattype, w_x=0.0): from pypy.objspace.std.floatobject import W_FloatObject @@ -10,7 +10,7 @@ if space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) @@ -21,7 +21,7 @@ from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/longobject.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/longobject.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/longobject.py Sat Dec 11 15:10:15 2010 @@ -45,19 +45,6 @@ fromrarith_int._annspecialcase_ = "specialize:argtype(0)" fromrarith_int = staticmethod(fromrarith_int) - def fromdecimalstr(s): - return W_LongObject(rbigint.fromdecimalstr(s)) - fromdecimalstr = staticmethod(fromdecimalstr) - - def _count_bits(self): - return self.num._count_bits() - - def is_odd(self): - return self.num.is_odd() - - def get_sign(self): - return self.num.sign - registerimplementation(W_LongObject) # bool-to-long Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/mapdict.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/mapdict.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/mapdict.py Sat Dec 11 15:10:15 2010 @@ -42,11 +42,23 @@ return None def index(self, selector): - if (self.space.config.objspace.std.withmethodcache and - not jit.we_are_jitted()): - return self._index_cache(selector) + if jit.we_are_jitted(): + # hack for the jit: + # the _index method is pure too, but its argument is never + # constant, because it is always a new tuple + return self._index_jit_pure(selector[0], selector[1]) else: - return self._index(selector) + return self._index_indirection(selector) + + @jit.purefunction + def _index_jit_pure(self, name, index): + return self._index_indirection((name, index)) + + @jit.dont_look_inside + def _index_indirection(self, selector): + if (self.space.config.objspace.std.withmethodcache): + return self._index_cache(selector) + return self._index(selector) @jit.dont_look_inside def _index_cache(self, selector): @@ -498,10 +510,11 @@ def _mapdict_read_storage(self, index): assert index >= 0 - for i in rangenmin1: - if index == i: - erased = getattr(self, "_value%s" % i) - return rerased.unerase(erased, W_Root) + if index < nmin1: + for i in rangenmin1: + if index == i: + erased = getattr(self, "_value%s" % i) + return rerased.unerase(erased, W_Root) if self._has_storage_list(): return self._mapdict_get_storage_list()[index - nmin1] erased = getattr(self, "_value%s" % nmin1) Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/objspace.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/objspace.py Sat Dec 11 15:10:15 2010 @@ -9,7 +9,7 @@ from pypy.objspace.descroperation import DescrOperation, raiseattrerror from pypy.rlib.objectmodel import instantiate, r_dict, specialize from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib.rarithmetic import base_int +from pypy.rlib.rarithmetic import base_int, widen from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.jit import hint from pypy.tool.sourcetools import func_with_new_name @@ -176,7 +176,11 @@ #print 'wrapping', x, '->', w_result return w_result if isinstance(x, base_int): - return W_LongObject.fromrarith_int(x) + x = widen(x) + if isinstance(x, int): + return self.newint(x) + else: + return W_LongObject.fromrarith_int(x) # _____ below here is where the annotator should not get _____ @@ -372,7 +376,7 @@ self, w_obj, expected_length)[:]) if expected_length != -1 and len(t) != expected_length: raise self._wrap_expected_length(expected_length, len(t)) - return t + return make_sure_not_resized(t) def fixedview_unroll(self, w_obj, expected_length=-1): return self.fixedview(w_obj, expected_length, unroll=True) Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/strutil.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/strutil.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/strutil.py Sat Dec 11 15:10:15 2010 @@ -150,7 +150,7 @@ del calc_mantissa_bits MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1 -def interp_string_to_float(space, s): +def string_to_float(s): """ Conversion of string to float. This version tries to only raise on invalid literals. @@ -162,10 +162,9 @@ s = strip_spaces(s) if not s: - raise OperationError(space.w_ValueError, space.wrap( - "empty string for float()")) + raise ParseStringError("empty string for float()") + - low = s.lower() if low == "-inf": return -INFINITY @@ -204,68 +203,56 @@ if len(digits) == 0: digits = '0' - # a few abbreviations - from pypy.objspace.std import longobject - mklong = longobject.W_LongObject.fromint - d2long = longobject.W_LongObject.fromdecimalstr - adlong = longobject.add__Long_Long - longup = longobject.pow__Long_Long_None - multip = longobject.mul__Long_Long - divide = longobject.div__Long_Long - lshift = longobject.lshift__Long_Long - rshift = longobject.rshift__Long_Long - # 4) compute the exponent and truncate to +-400 if not exponent: exponent = '0' - w_le = d2long(exponent) - w_le = adlong(space, w_le, mklong(space, dexp)) + long_exponent = rbigint.fromdecimalstr(exponent) + long_exponent = long_exponent.add(rbigint.fromint(dexp)) try: - e = w_le.toint() + e = long_exponent.toint() except OverflowError: # XXX poking at internals - e = w_le.num.sign * 400 - if e >= 400: - e = 400 - elif e <= -400: - e = -400 + e = long_exponent.sign * 400 + else: + if e >= 400: + e = 400 + elif e <= -400: + e = -400 # 5) compute the value using long math and proper rounding. - w_lr = d2long(digits) - w_10 = mklong(space, 10) - w_1 = mklong(space, 1) + b_digits = rbigint.fromdecimalstr(digits) + b_10 = rbigint.fromint(10) + b_1 = rbigint.fromint(1) if e >= 0: bits = 0 - w_pten = longup(space, w_10, mklong(space, e), space.w_None) - w_m = multip(space, w_lr, w_pten) + b_power_of_ten = b_10.pow(rbigint.fromint(e)) + b_mantissa = b_digits.mul(b_power_of_ten) else: # compute a sufficiently large scale prec = MANTISSA_DIGITS * 2 + 22 # 128, maybe bits = - (int(math.ceil(-e / math.log10(2.0) - 1e-10)) + prec) - w_scale = lshift(space, w_1, mklong(space, -bits)) - w_pten = longup(space, w_10, mklong(space, -e), None) - w_tmp = multip(space, w_lr, w_scale) - w_m = divide(space, w_tmp, w_pten) + b_scale = b_1.lshift(-bits) + b_power_of_ten = b_10.pow(rbigint.fromint(-e)) + b_mantissa = b_digits.mul(b_scale).div(b_power_of_ten) # we now have a fairly large mantissa. # Shift it and round the last bit. # first estimate the bits and do a big shift - mbits = w_m._count_bits() + mbits = b_mantissa._count_bits() needed = MANTISSA_BITS if mbits > needed: if mbits > needed+1: shifted = mbits - (needed+1) - w_m = rshift(space, w_m, mklong(space, shifted)) + b_mantissa = b_mantissa.rshift(shifted) bits += shifted # do the rounding bits += 1 - round = w_m.is_odd() - w_m = rshift(space, w_m, w_1) - w_m = adlong(space, w_m, mklong(space, round)) + round = b_mantissa.is_odd() + b_mantissa = b_mantissa.rshift(1).add(rbigint.fromint(round)) try: - r = math.ldexp(w_m.tofloat(), bits) + r = math.ldexp(b_mantissa.tofloat(), bits) # XXX I guess we do not check for overflow in ldexp as we agreed to! if r == 2*r and r != 0.0: raise OverflowError Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/test/test_complexobject.py Sat Dec 11 15:10:15 2010 @@ -1,5 +1,6 @@ import py -from pypy.objspace.std import complexobject as cobj +from pypy.objspace.std.complexobject import W_ComplexObject, \ + pow__Complex_Complex_ANY from pypy.objspace.std import complextype as cobjtype from pypy.objspace.std.multimethod import FailedToImplement from pypy.objspace.std.stringobject import W_StringObject @@ -11,7 +12,7 @@ def _test_instantiation(self): def _t_complex(r=0.0,i=0.0): - c = cobj.W_ComplexObject(r, i) + c = W_ComplexObject(r, i) assert c.real == float(r) and c.imag == float(i) pairs = ( (1, 1), @@ -38,21 +39,31 @@ test_cparse('.e+5', '.e+5', '0.0') def test_pow(self): - assert cobj._pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) - assert cobj._pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) - rr, ir = cobj._pow((0.0,1.0),(2.0,0.0)) + def _pow((r1, i1), (r2, i2)): + w_res = W_ComplexObject(r1, i1).pow(W_ComplexObject(r2, i2)) + return w_res.realval, w_res.imagval + assert _pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) + assert _pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) + rr, ir = _pow((0.0,1.0),(2.0,0.0)) assert abs(-1.0 - rr) < EPS assert abs(0.0 - ir) < EPS - assert cobj._powu((0.0,2.0),0) == (1.0,0.0) - assert cobj._powu((0.0,0.0),2) == (0.0,0.0) - assert cobj._powu((0.0,1.0),2) == (-1.0,0.0) - assert cobj._powi((0.0,2.0),0) == (1.0,0.0) - assert cobj._powi((0.0,0.0),2) == (0.0,0.0) - assert cobj._powi((0.0,1.0),2) == (-1.0,0.0) - c = cobj.W_ComplexObject(0.0,1.0) - p = cobj.W_ComplexObject(2.0,0.0) - r = cobj.pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) + def _powu((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_positive_int(n) + return w_res.realval, w_res.imagval + assert _powu((0.0,2.0),0) == (1.0,0.0) + assert _powu((0.0,0.0),2) == (0.0,0.0) + assert _powu((0.0,1.0),2) == (-1.0,0.0) + + def _powi((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_int(n) + return w_res.realval, w_res.imagval + assert _powi((0.0,2.0),0) == (1.0,0.0) + assert _powi((0.0,0.0),2) == (0.0,0.0) + assert _powi((0.0,1.0),2) == (-1.0,0.0) + c = W_ComplexObject(0.0,1.0) + p = W_ComplexObject(2.0,0.0) + r = pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) assert r.realval == -1.0 assert r.imagval == 0.0 Modified: pypy/branch/jit-unroll-loops/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/branch/jit-unroll-loops/pypy/objspace/std/test/test_strutil.py Sat Dec 11 15:10:15 2010 @@ -131,8 +131,6 @@ assert string_to_bigint('1891234174197319').tolong() == 1891234174197319 def test_string_to_float(self): - def string_to_float(x): - return interp_string_to_float(self.space, x) assert string_to_float('0') == 0.0 assert string_to_float('1') == 1.0 assert string_to_float('-1.5') == -1.5 @@ -180,3 +178,4 @@ print repr(s) if s.strip(): # empty s raises OperationError directly py.test.raises(ParseStringError, string_to_float, s) + py.test.raises(ParseStringError, string_to_float, "") Modified: pypy/branch/jit-unroll-loops/pypy/rlib/debug.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/debug.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/debug.py Sat Dec 11 15:10:15 2010 @@ -53,13 +53,11 @@ _log = None # patched from tests to be an object of class DebugLog # or compatible -_stderr = sys.stderr # alternatively, this is patched from tests - # (redirects debug_print(), but not debug_start/stop) def debug_print(*args): for arg in args: - print >> _stderr, arg, - print >> _stderr + print >> sys.stderr, arg, + print >> sys.stderr if _log is not None: _log.debug_print(*args) @@ -87,13 +85,15 @@ _stop_colors = "" def debug_start(category): - print >> sys.stderr, '%s[%s] {%s%s' % (_start_colors_1, time.clock(), + c = int(time.clock() * 100) + print >> sys.stderr, '%s[%x] {%s%s' % (_start_colors_1, c, category, _stop_colors) if _log is not None: _log.debug_start(category) def debug_stop(category): - print >> sys.stderr, '%s[%s] %s}%s' % (_start_colors_2, time.clock(), + c = int(time.clock() * 100) + print >> sys.stderr, '%s[%x] %s}%s' % (_start_colors_2, c, category, _stop_colors) if _log is not None: _log.debug_stop(category) @@ -226,31 +226,6 @@ hop.exception_cannot_occur() return hop.inputarg(hop.args_r[0], arg=0) -def make_sure_not_modified(arg): - """ Function checking whether annotation of SomeList is never resized - and never modified, useful for debugging. Does nothing when run directly - """ - return arg - -class Entry(ExtRegistryEntry): - _about_ = make_sure_not_modified - - def compute_result_annotation(self, s_arg): - from pypy.annotation.model import SomeList - assert isinstance(s_arg, SomeList) - # the logic behind it is that we try not to propagate - # make_sure_not_resized, when list comprehension is not on - if self.bookkeeper.annotator.translator.config.translation.list_comprehension_operations: - s_arg.listdef.never_mutate() - else: - from pypy.annotation.annrpython import log - log.WARNING('make_sure_not_modified called, but has no effect since list_comprehension is off') - return s_arg - - def specialize_call(self, hop): - hop.exception_cannot_occur() - return hop.inputarg(hop.args_r[0], arg=0) - class IntegerCanBeNegative(Exception): pass Modified: pypy/branch/jit-unroll-loops/pypy/rlib/jit.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/jit.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/jit.py Sat Dec 11 15:10:15 2010 @@ -156,7 +156,7 @@ def jit_debug(string, arg1=-sys.maxint-1, arg2=-sys.maxint-1, arg3=-sys.maxint-1, arg4=-sys.maxint-1): - """When JITted, cause an extra operation DEBUG_MERGE_POINT to appear in + """When JITted, cause an extra operation JIT_DEBUG to appear in the graphs. Should not be left after debugging.""" keepalive_until_here(string) # otherwise the whole function call is removed jit_debug.oopspec = 'jit.debug(string, arg1, arg2, arg3, arg4)' @@ -260,18 +260,12 @@ OPTIMIZER_NO_UNROLL = 1 OPTIMIZER_FULL = 2 -DEBUG_OFF = 0 -DEBUG_PROFILE = 1 -DEBUG_STEPS = 2 -DEBUG_DETAILED = 3 - PARAMETERS = {'threshold': 1000, 'trace_eagerness': 200, 'trace_limit': 10000, 'inlining': False, 'optimizer': OPTIMIZER_FULL, - #'optimizer': OPTIMIZER_NO_UNROLL, - 'debug' : DEBUG_STEPS, + 'loop_longevity': 1000, } unroll_parameters = unrolling_iterable(PARAMETERS.keys()) Modified: pypy/branch/jit-unroll-loops/pypy/rlib/libffi.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/libffi.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/libffi.py Sat Dec 11 15:10:15 2010 @@ -178,6 +178,9 @@ # the optimizer will fail to recognize the pattern and won't turn it # into a fast CALL. Note that "arg = arg.next" is optimized away, # assuming that archain is completely virtual. + if argchain.numargs != len(self.argtypes): + raise TypeError, 'Wrong number of arguments: %d expected, got %d' %\ + (argchain.numargs, len(self.argtypes)) ll_args = self._prepare() i = 0 arg = argchain.first Modified: pypy/branch/jit-unroll-loops/pypy/rlib/rarithmetic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/rarithmetic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/rarithmetic.py Sat Dec 11 15:10:15 2010 @@ -92,7 +92,8 @@ return False r_class = rffi.platform.numbertype_to_rclass[tp] assert issubclass(r_class, base_int) - return r_class.BITS < LONG_BIT + return r_class.BITS < LONG_BIT or ( + r_class.BITS == LONG_BIT and r_class.SIGNED) _should_widen_type._annspecialcase_ = 'specialize:memo' del _bits, _itest, _Ltest @@ -389,6 +390,11 @@ r_longlong = build_int('r_longlong', True, 64) r_ulonglong = build_int('r_ulonglong', False, 64) +if r_longlong is not r_int: + r_int64 = r_longlong +else: + r_int64 = int + # float as string -> sign, beforept, afterpt, exponent Modified: pypy/branch/jit-unroll-loops/pypy/rlib/rdynload.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/rdynload.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/rdynload.py Sat Dec 11 15:10:15 2010 @@ -14,7 +14,7 @@ _MINGW = platform.name == "mingw32" _WIN32 = _MSVC or _MINGW _MAC_OS = platform.name == "darwin" -_FREEBSD_7 = platform.name == "freebsd7" +_FREEBSD = platform.name == "freebsd" if _WIN32: from pypy.rlib import rwin32 @@ -27,7 +27,7 @@ else: pre_include_bits = [] -if _FREEBSD_7 or _WIN32: +if _FREEBSD or _WIN32: libraries = [] else: libraries = ['dl'] Modified: pypy/branch/jit-unroll-loops/pypy/rlib/rerased.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/rerased.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/rerased.py Sat Dec 11 15:10:15 2010 @@ -91,7 +91,7 @@ return annmodel.SomeInteger() assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() return annmodel.SomeInstance(clsdef) def specialize_call(self, hop): @@ -108,7 +108,7 @@ def compute_result_annotation(self, s_obj, s_type): assert isinstance(s_type, annmodel.SomePBC) assert len(s_type.descriptions) == 1 - clsdef = s_type.descriptions.keys()[0].getuniqueclassdef() + clsdef = s_type.any_description().getuniqueclassdef() s_item = annmodel.SomeInstance(clsdef) return self.bookkeeper.newlist(s_item) Modified: pypy/branch/jit-unroll-loops/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/rmmap.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/rmmap.py Sat Dec 11 15:10:15 2010 @@ -67,7 +67,7 @@ constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE', - 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE'] + 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS'] for name in constant_names: setattr(CConfig, name, rffi_platform.ConstantInteger(name)) @@ -100,8 +100,11 @@ sandboxsafe=True, threadsafe=False) return unsafe, safe -def winexternal(name, args, result): - return rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_, calling_conv='win') +def winexternal(name, args, result, **kwargs): + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_, + calling_conv='win', + **kwargs) PTR = rffi.CCHARP @@ -189,9 +192,17 @@ VirtualAlloc = winexternal('VirtualAlloc', [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD], rffi.VOIDP) - VirtualProtect = winexternal('VirtualProtect', - [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], - BOOL) + # VirtualProtect is used in llarena and should not release the GIL + _VirtualProtect = winexternal('VirtualProtect', + [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], + BOOL, + _nowrapper=True) + def VirtualProtect(addr, size, mode, oldmode_ptr): + return _VirtualProtect(addr, + rffi.cast(rffi.SIZE_T, size), + rffi.cast(DWORD, mode), + oldmode_ptr) + VirtualProtect._annspecialcase_ = 'specialize:ll' VirtualFree = winexternal('VirtualFree', [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL) Modified: pypy/branch/jit-unroll-loops/pypy/rlib/rsre/rsre_core.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/rsre/rsre_core.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/rsre/rsre_core.py Sat Dec 11 15:10:15 2010 @@ -1,5 +1,5 @@ import sys -from pypy.rlib.debug import check_nonneg, make_sure_not_modified +from pypy.rlib.debug import check_nonneg from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.rsre import rsre_char from pypy.tool.sourcetools import func_with_new_name @@ -471,7 +471,6 @@ while True: op = ctx.pat(ppos) ppos += 1 - make_sure_not_modified(ctx.pattern) #jit.jit_debug("sre_match", op, ppos, ptr) # Modified: pypy/branch/jit-unroll-loops/pypy/rlib/rsre/test/test_zjit.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/rsre/test/test_zjit.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/rsre/test/test_zjit.py Sat Dec 11 15:10:15 2010 @@ -1,6 +1,5 @@ from pypy.jit.metainterp.test import test_basic from pypy.rlib.nonconst import NonConstant -from pypy.rlib.debug import make_sure_not_modified from pypy.rlib.rsre.test.test_match import get_code from pypy.rlib.rsre import rsre_core from pypy.rpython.lltypesystem import lltype Modified: pypy/branch/jit-unroll-loops/pypy/rlib/test/test_debug.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/test/test_debug.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/test/test_debug.py Sat Dec 11 15:10:15 2010 @@ -42,14 +42,14 @@ py.test.raises(IntegerCanBeNegative, interpret, g, [9]) def test_make_sure_not_resized(): - from pypy.annotation.listdef import TooLateForChange + from pypy.annotation.listdef import ListChangeUnallowed def f(): result = [1,2,3] make_sure_not_resized(result) result.append(4) return len(result) - py.test.raises(TooLateForChange, interpret, f, [], + py.test.raises(ListChangeUnallowed, interpret, f, [], list_comprehension_operations=True) Modified: pypy/branch/jit-unroll-loops/pypy/rlib/test/test_libffi.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rlib/test/test_libffi.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rlib/test/test_libffi.py Sat Dec 11 15:10:15 2010 @@ -262,3 +262,24 @@ # res = self.call(get_dummy, [], rffi.LONG) assert res == initval+1 + + def test_wrong_number_of_arguments(self): + from pypy.rpython.llinterp import LLException + libfoo = self.get_libfoo() + func = (libfoo, 'sum_xy', [types.sint, types.double], types.sint) + + glob = globals() + loc = locals() + def my_raises(s): + try: + exec s in glob, loc + except TypeError: + pass + except LLException, e: + if str(e) != "": + raise + else: + assert False, 'Did not raise' + + my_raises("self.call(func, [38], rffi.LONG)") # one less + my_raises("self.call(func, [38, 12.3, 42], rffi.LONG)") # one more Modified: pypy/branch/jit-unroll-loops/pypy/rpython/extfunc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/extfunc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/extfunc.py Sat Dec 11 15:10:15 2010 @@ -201,6 +201,11 @@ exec py.code.compile(""" from pypy.rlib.objectmodel import running_on_llinterp from pypy.rlib.debug import llinterpcall + from pypy.rlib.jit import dont_look_inside + # note: we say 'dont_look_inside' mostly because the + # JIT does not support 'running_on_llinterp', but in + # theory it is probably right to stop jitting anyway. + @dont_look_inside def ll_wrapper(%s): if running_on_llinterp: return llinterpcall(s_result, fakeimpl, %s) Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/ll2ctypes.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/ll2ctypes.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/ll2ctypes.py Sat Dec 11 15:10:15 2010 @@ -409,6 +409,7 @@ subcls = get_common_subclass(mixin_cls, instance.__class__) instance.__class__ = subcls instance._storage = ctypes_storage + assert ctypes_storage # null pointer? class _parentable_mixin(object): """Mixin added to _parentable containers when they become ctypes-based. @@ -442,6 +443,9 @@ "not allocated from RPython at all") self._storage = None + def _getid(self): + return self._addressof_storage() + def __eq__(self, other): if isinstance(other, _llgcopaque): addressof_other = other.intval @@ -1291,7 +1295,7 @@ def _where_is_errno(): return standard_c_lib.__errno_location() - elif sys.platform in ('darwin', 'freebsd7'): + elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'): standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int) def _where_is_errno(): return standard_c_lib.__error() Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/llarena.py Sat Dec 11 15:10:15 2010 @@ -26,6 +26,7 @@ self.objectptrs = {} # {offset: ptr-to-container} self.objectsizes = {} # {offset: size} self.freed = False + self.protect_inaccessible = None self.reset(zero) def __repr__(self): @@ -59,6 +60,8 @@ def check(self): if self.freed: raise ArenaError("arena was already freed") + if self.protect_inaccessible is not None: + raise ArenaError("arena is currently arena_protect()ed") def _getid(self): address, length = self.usagemap.buffer_info() @@ -127,6 +130,21 @@ def mark_freed(self): self.freed = True # this method is a hook for tests + def set_protect(self, inaccessible): + if inaccessible: + assert self.protect_inaccessible is None + saved = [] + for ptr in self.objectptrs.values(): + obj = ptr._obj + saved.append((obj, obj._protect())) + self.protect_inaccessible = saved + else: + assert self.protect_inaccessible is not None + saved = self.protect_inaccessible + for obj, storage in saved: + obj._unprotect(storage) + self.protect_inaccessible = None + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -365,6 +383,16 @@ """ return Arena(ptr.arena.nbytes, False).getaddr(0) +def arena_protect(arena_addr, size, inaccessible): + """For debugging, set or reset memory protection on an arena. + For now, the starting point and size should reference the whole arena. + The value of 'inaccessible' is a boolean. + """ + arena_addr = getfakearenaaddress(arena_addr) + assert arena_addr.offset == 0 + assert size == arena_addr.arena.nbytes + arena_addr.arena.set_protect(inaccessible) + # ____________________________________________________________ # # Translation support: the functions above turn into the code below. @@ -475,6 +503,42 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear +if os.name == "posix": + from pypy.translator.tool.cbuild import ExternalCompilationInfo + _eci = ExternalCompilationInfo(includes=['sys/mman.h']) + raw_mprotect = rffi.llexternal('mprotect', + [llmemory.Address, rffi.SIZE_T, rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True, + compilation_info=_eci) + def llimpl_protect(addr, size, inaccessible): + if inaccessible: + prot = 0 + else: + from pypy.rlib.rmmap import PROT_READ, PROT_WRITE + prot = PROT_READ | PROT_WRITE + raw_mprotect(addr, rffi.cast(rffi.SIZE_T, size), + rffi.cast(rffi.INT, prot)) + # ignore potential errors + has_protect = True + +elif os.name == 'nt': + def llimpl_protect(addr, size, inaccessible): + from pypy.rlib.rmmap import VirtualProtect, LPDWORD + if inaccessible: + from pypy.rlib.rmmap import PAGE_NOACCESS as newprotect + else: + from pypy.rlib.rmmap import PAGE_READWRITE as newprotect + arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') + VirtualProtect(rffi.cast(rffi.VOIDP, addr), + size, newprotect, arg) + # ignore potential errors + lltype.free(arg, flavor='raw') + has_protect = True + +else: + has_protect = False + llimpl_malloc = rffi.llexternal('malloc', [lltype.Signed], llmemory.Address, sandboxsafe=True, _nowrapper=True) @@ -544,6 +608,21 @@ 'll_arena.arena_new_view', llimpl=llimpl_arena_new_view, llfakeimpl=arena_new_view, sandboxsafe=True) +def llimpl_arena_protect(addr, size, inaccessible): + if has_protect: + # do some alignment + start = rffi.cast(lltype.Signed, addr) + end = start + size + start = (start + 4095) & ~ 4095 + end = end & ~ 4095 + if end > start: + llimpl_protect(rffi.cast(llmemory.Address, start), end-start, + inaccessible) +register_external(arena_protect, [llmemory.Address, lltype.Signed, + lltype.Bool], lltype.Void, + 'll_arena.arena_protect', llimpl=llimpl_arena_protect, + llfakeimpl=arena_protect, sandboxsafe=True) + def llimpl_getfakearenaaddress(addr): return addr register_external(getfakearenaaddress, [llmemory.Address], llmemory.Address, Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/llmemory.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/llmemory.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/llmemory.py Sat Dec 11 15:10:15 2010 @@ -93,8 +93,10 @@ return endmarker._as_ptr() else: return parent.getitem(index)._as_ptr() - elif (isinstance(A, lltype.FixedSizeArray) and - array_item_type_match(A.OF, self.TYPE)): + elif ((isinstance(A, lltype.FixedSizeArray) + or (isinstance(A, lltype.Array) and A._hints.get('nolength', + False))) + and array_item_type_match(A.OF, self.TYPE)): # for array of primitives or pointers return lltype.direct_ptradd(firstitemptr, self.repeat) else: Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/lltype.py Sat Dec 11 15:10:15 2010 @@ -1381,6 +1381,15 @@ self._check() # no double-frees self._storage = None + def _protect(self): + result = self._storage + self._free() # no double-frees or double-protects + return result + + def _unprotect(self, saved_storage): + assert self._storage is None + self._storage = saved_storage + def _was_freed(self): if self._storage is None: return True Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rbuiltin.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rbuiltin.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rbuiltin.py Sat Dec 11 15:10:15 2010 @@ -42,7 +42,7 @@ return hop.genop('cast_pointer', [v_inst], # v_type implicit in r_result resulttype = hop.r_result.lowleveltype) - classdef = s_class.descriptions.keys()[0].getuniqueclassdef() + classdef = s_class.any_description().getuniqueclassdef() return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops) def rtype_builtin_hasattr(hop): Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rclass.py Sat Dec 11 15:10:15 2010 @@ -392,7 +392,7 @@ source_classdef = source_desc.getclassdef(None) source_repr = getinstancerepr(self.rtyper, source_classdef) assert len(s_func.descriptions) == 1 - funcdesc = s_func.descriptions.keys()[0] + funcdesc, = s_func.descriptions graph = funcdesc.getuniquegraph() FUNCTYPE = FuncType([Ptr(source_repr.object_type)], Void) destrptr = functionptr(FUNCTYPE, graph.name, Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rdict.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rdict.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rdict.py Sat Dec 11 15:10:15 2010 @@ -581,7 +581,7 @@ def ll_dict_lookup_clean(d, hash): # a simplified version of ll_dict_lookup() which assumes that the # key is new, and the dictionary doesn't contain deleted entries. - # It only find the next free slot for the given hash. + # It only finds the next free slot for the given hash. entries = d.entries mask = len(entries) - 1 i = hash & mask Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rpbc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rpbc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/rpbc.py Sat Dec 11 15:10:15 2010 @@ -127,7 +127,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(self.rtyper, self.callfamily) assert len(uniquerows) == 1 @@ -166,7 +166,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) def convert_desc(self, funcdesc): @@ -230,7 +230,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness Modified: pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/lltypesystem/test/test_llarena.py Sat Dec 11 15:10:15 2010 @@ -6,6 +6,8 @@ from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view from pypy.rpython.lltypesystem.llarena import arena_shrink_obj +from pypy.rpython.lltypesystem.llarena import arena_protect, has_protect +from pypy.translator.c.test import test_genc, test_standalone def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -265,8 +267,7 @@ assert res == 42 def test_compiled(): - from pypy.translator.c.test.test_genc import compile - fn = compile(test_look_inside_object, []) + fn = test_genc.compile(test_look_inside_object, []) res = fn() assert res == 42 @@ -282,3 +283,51 @@ arena_reserve(a, size_gc_header + llmemory.sizeof(S, 10)) arena_shrink_obj(a, size_gc_header + llmemory.sizeof(S, 5)) arena_reset(a, size_gc_header + llmemory.sizeof(S, 5), False) + +def test_arena_protect(): + a = arena_malloc(100, False) + S = lltype.Struct('S', ('x', lltype.Signed)) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 100, True) + py.test.raises(ArenaError, arena_reserve, a + 48, llmemory.sizeof(S)) + py.test.raises(RuntimeError, "p.x") + py.test.raises(RuntimeError, "p.x = 124") + arena_protect(a, 100, False) + assert p.x == 123 + p.x = 125 + assert p.x == 125 + + +class TestStandalone(test_standalone.StandaloneTests): + def test_compiled_arena_protect(self): + import os + from pypy.translator.c.test.test_genc import compile + S = lltype.Struct('S', ('x', lltype.Signed)) + # + def fn(argv): + testrun = int(argv[1]) + a = arena_malloc(65536, False) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a + 23432, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 65536, True) + result = 0 + if testrun == 1: + print p.x # segfault + if testrun == 2: + p.x = 124 # segfault + arena_protect(a, 65536, False) + p.x += 10 + print p.x + return 0 + # + t, cbuilder = self.compile(fn) + data = cbuilder.cmdexec('0') + assert data == '133\n' + if has_protect: + cbuilder.cmdexec('1', expect_crash=True) + cbuilder.cmdexec('2', expect_crash=True) Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/base.py Sat Dec 11 15:10:15 2010 @@ -5,7 +5,6 @@ from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage -from pypy.rlib.rarithmetic import r_uint TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed), ('size', lltype.Signed), @@ -19,6 +18,7 @@ malloc_zero_filled = False prebuilt_gc_objects_are_static_roots = True object_minimal_size = 0 + gcflag_extra = 0 # or a real GC flag that is always 0 when not collecting def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE, translated_to_c=True): @@ -36,6 +36,12 @@ self.finalizer_lock_count = 0 self.run_finalizers = self.AddressDeque() + def post_setup(self): + # More stuff that needs to be initialized when the GC is already + # fully working. (Only called by gctransform/framework for now.) + from pypy.rpython.memory.gc import env + self.DEBUG = env.read_from_env('PYPY_GC_DEBUG') + def _teardown(self): pass @@ -48,7 +54,8 @@ # The following flag enables costly consistency checks after each # collection. It is automatically set to True by test_gc.py. The # checking logic is translatable, so the flag can be set to True - # here before translation. + # here before translation. At run-time, if PYPY_GC_DEBUG is set, + # then it is also set to True. DEBUG = False def set_query_functions(self, is_varsize, has_gcptr_in_varsize, @@ -410,42 +417,6 @@ GCClass = getattr(module, classname) return GCClass, GCClass.TRANSLATION_PARAMS -def _read_float_and_factor_from_env(varname): - import os - value = os.environ.get(varname) - if value: - if len(value) > 1 and value[-1] in 'bB': - value = value[:-1] - realvalue = value[:-1] - if value[-1] in 'kK': - factor = 1024 - elif value[-1] in 'mM': - factor = 1024*1024 - elif value[-1] in 'gG': - factor = 1024*1024*1024 - else: - factor = 1 - realvalue = value - try: - return (float(realvalue), factor) - except ValueError: - pass - return (0.0, 0) - -def read_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - return int(value * factor) - -def read_uint_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - return r_uint(value * factor) - -def read_float_from_env(varname): - value, factor = _read_float_and_factor_from_env(varname) - if factor != 1: - return 0.0 - return value - def _convert_callback_formats(callback): callback = getattr(callback, 'im_func', callback) if callback not in _converted_callback_formats: Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/generation.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/generation.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/generation.py Sat Dec 11 15:10:15 2010 @@ -2,7 +2,7 @@ from pypy.rpython.memory.gc.semispace import SemiSpaceGC from pypy.rpython.memory.gc.semispace import GCFLAG_EXTERNAL, GCFLAG_FORWARDED from pypy.rpython.memory.gc.semispace import GC_HASH_TAKEN_ADDR -from pypy.rpython.memory.gc.base import read_from_env +from pypy.rpython.memory.gc import env from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage from pypy.rpython.lltypesystem import lltype, llmemory, llarena from pypy.rlib.objectmodel import free_non_gc_object @@ -93,7 +93,7 @@ if self.auto_nursery_size: newsize = nursery_size_from_env() if newsize <= 0: - newsize = estimate_best_nursery_size() + newsize = env.estimate_best_nursery_size() if newsize > 0: self.set_nursery_size(newsize) @@ -633,139 +633,5 @@ # ____________________________________________________________ -import os - def nursery_size_from_env(): - return read_from_env('PYPY_GENERATIONGC_NURSERY') - -def best_nursery_size_for_L2cache(L2cache): - # Heuristically, the best nursery size to choose is about half - # of the L2 cache. XXX benchmark some more. - return L2cache // 2 - - -if sys.platform == 'linux2': - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. - """ - debug_start("gc-L2cache") - L2cache = sys.maxint - try: - fd = os.open('/proc/cpuinfo', os.O_RDONLY, 0644) - try: - data = [] - while True: - buf = os.read(fd, 4096) - if not buf: - break - data.append(buf) - finally: - os.close(fd) - except OSError: - pass - else: - data = ''.join(data) - linepos = 0 - while True: - start = findend(data, '\ncache size', linepos) - if start < 0: - break # done - linepos = findend(data, '\n', start) - if linepos < 0: - break # no end-of-line?? - # *** data[start:linepos] == " : 2048 KB\n" - start = skipspace(data, start) - if data[start] != ':': - continue - # *** data[start:linepos] == ": 2048 KB\n" - start = skipspace(data, start + 1) - # *** data[start:linepos] == "2048 KB\n" - end = start - while '0' <= data[end] <= '9': - end += 1 - # *** data[start:end] == "2048" - if start == end: - continue - number = int(data[start:end]) - # *** data[end:linepos] == " KB\n" - end = skipspace(data, end) - if data[end] not in ('K', 'k'): # assume kilobytes for now - continue - number = number * 1024 - # for now we look for the smallest of the L2 caches of the CPUs - if number < L2cache: - L2cache = number - - debug_print("L2cache =", L2cache) - debug_stop("gc-L2cache") - - if L2cache < sys.maxint: - return best_nursery_size_for_L2cache(L2cache) - else: - # Print a top-level warning even in non-debug builds - llop.debug_print(lltype.Void, - "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") - return -1 - - def findend(data, pattern, pos): - pos = data.find(pattern, pos) - if pos < 0: - return -1 - return pos + len(pattern) - - def skipspace(data, pos): - while data[pos] in (' ', '\t'): - pos += 1 - return pos - -elif sys.platform == 'darwin': - from pypy.rpython.lltypesystem import rffi - - sysctlbyname = rffi.llexternal('sysctlbyname', - [rffi.CCHARP, rffi.VOIDP, rffi.SIZE_TP, - rffi.VOIDP, rffi.SIZE_T], - rffi.INT, - sandboxsafe=True) - - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. - """ - debug_start("gc-L2cache") - L2cache = 0 - l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') - try: - len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') - try: - size = rffi.sizeof(rffi.LONGLONG) - l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) - len_p[0] = rffi.cast(rffi.SIZE_T, size) - # XXX a hack for llhelper not being robust-enough - result = sysctlbyname("hw.l2cachesize", - rffi.cast(rffi.VOIDP, l2cache_p), - len_p, - lltype.nullptr(rffi.VOIDP.TO), - rffi.cast(rffi.SIZE_T, 0)) - if (rffi.cast(lltype.Signed, result) == 0 and - rffi.cast(lltype.Signed, len_p[0]) == size): - L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) - if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: - L2cache = 0 # overflow! - finally: - lltype.free(len_p, flavor='raw') - finally: - lltype.free(l2cache_p, flavor='raw') - debug_print("L2cache =", L2cache) - debug_stop("gc-L2cache") - if L2cache > 0: - return best_nursery_size_for_L2cache(L2cache) - else: - # Print a top-level warning even in non-debug builds - llop.debug_print(lltype.Void, - "Warning: cannot find your CPU L2 cache size with sysctl()") - return -1 - -else: - def estimate_best_nursery_size(): - return -1 # XXX implement me for other platforms + return env.read_from_env('PYPY_GENERATIONGC_NURSERY') Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/inspector.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/inspector.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/inspector.py Sat Dec 11 15:10:15 2010 @@ -101,21 +101,24 @@ AddressStack = get_address_stack() -class HeapDumper: +class HeapDumper(object): _alloc_flavor_ = "raw" BUFSIZE = 8192 # words def __init__(self, gc, fd): self.gc = gc + self.gcflag = gc.gcflag_extra self.fd = rffi.cast(rffi.INT, fd) self.writebuffer = lltype.malloc(rffi.LONGP.TO, self.BUFSIZE, flavor='raw') self.buf_count = 0 - self.seen = AddressDict() + if self.gcflag == 0: + self.seen = AddressDict() self.pending = AddressStack() def delete(self): - self.seen.delete() + if self.gcflag == 0: + self.seen.delete() self.pending.delete() lltype.free(self.writebuffer, flavor='raw') free_non_gc_object(self) @@ -140,6 +143,8 @@ self.flush() write._always_inline_ = True + # ---------- + def write_marker(self): self.write(0) self.write(0) @@ -161,9 +166,15 @@ self.add(obj) def add(self, obj): - if not self.seen.contains(obj): - self.seen.setitem(obj, obj) - self.pending.append(obj) + if self.gcflag == 0: + if not self.seen.contains(obj): + self.seen.setitem(obj, obj) + self.pending.append(obj) + else: + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) == 0: + hdr.tid |= self.gcflag + self.pending.append(obj) def add_roots(self): self.gc.enumerate_all_roots(_hd_add_root, self) @@ -177,14 +188,50 @@ while pending.non_empty(): self.writeobj(pending.pop()) + # ---------- + # A simplified copy of the above, to make sure we walk again all the + # objects to clear the 'gcflag'. + + def unwriteobj(self, obj): + gc = self.gc + gc.trace(obj, self._unwriteref, None) + + def _unwriteref(self, pointer, _): + obj = pointer.address[0] + self.unadd(obj) + + def unadd(self, obj): + assert self.gcflag != 0 + hdr = self.gc.header(obj) + if (hdr.tid & self.gcflag) != 0: + hdr.tid &= ~self.gcflag + self.pending.append(obj) + + def clear_gcflag_again(self): + self.gc.enumerate_all_roots(_hd_unadd_root, self) + pendingroots = self.pending + self.pending = AddressStack() + self.unwalk(pendingroots) + pendingroots.delete() + + def unwalk(self, pending): + while pending.non_empty(): + self.unwriteobj(pending.pop()) + def _hd_add_root(obj, heap_dumper): heap_dumper.add(obj) +def _hd_unadd_root(obj, heap_dumper): + heap_dumper.unadd(obj) + def dump_rpy_heap(gc, fd): heapdumper = HeapDumper(gc, fd) heapdumper.add_roots() heapdumper.walk(heapdumper.pending) heapdumper.flush() + if heapdumper.gcflag != 0: + heapdumper.clear_gcflag_again() + heapdumper.unwalk(heapdumper.pending) heapdumper.delete() return True Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/markcompact.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/markcompact.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/markcompact.py Sat Dec 11 15:10:15 2010 @@ -1,5 +1,6 @@ from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup -from pypy.rpython.memory.gc.base import MovingGCBase, read_from_env +from pypy.rpython.memory.gc.base import MovingGCBase +from pypy.rpython.memory.gc import env from pypy.rlib.debug import ll_assert, have_debug_prints from pypy.rlib.debug import debug_print, debug_start, debug_stop from pypy.rpython.memory.support import get_address_stack, get_address_deque @@ -110,10 +111,10 @@ return next def setup(self): - envsize = read_from_env('PYPY_MARKCOMPACTGC_MAX') + envsize = env.read_from_env('PYPY_MARKCOMPACTGC_MAX') if envsize >= 4096: self.space_size = envsize & ~4095 - mincollect = read_from_env('PYPY_MARKCOMPACTGC_MIN') + mincollect = env.read_from_env('PYPY_MARKCOMPACTGC_MIN') if mincollect >= 4096: self.min_next_collect_after = mincollect Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/minimark.py Sat Dec 11 15:10:15 2010 @@ -3,7 +3,8 @@ Environment variables can be used to fine-tune the following parameters: PYPY_GC_NURSERY The nursery size. Defaults to half the size of - the L2 cache. Try values like '1.2MB'. + the L2 cache. Try values like '1.2MB'. Small values + (like 1 or 1KB) are useful for debugging. PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', which means trigger a major collection when the @@ -12,7 +13,7 @@ collection. PYPY_GC_GROWTH Major collection threshold's max growth rate. - Default is '1.3'. Useful to collect more often + Default is '1.4'. Useful to collect more often than normally on sudden memory growth, e.g. when there is a temporary peak in memory usage. @@ -22,10 +23,21 @@ crash the program with a fatal error. Try values like '1.6GB'. + PYPY_GC_MAX_DELTA The major collection threshold will never be set + to more than PYPY_GC_MAX_DELTA the amount really + used after a collection. Defaults to 1/8th of the + total RAM size (which is constrained to be at most + 2/3/4GB on 32-bit systems). Try values like '200MB'. + PYPY_GC_MIN Don't collect while the memory size is below this limit. Useful to avoid spending all the time in the GC in very small programs. Defaults to 8 times the nursery. + + PYPY_GC_DEBUG Enable extra checks around collections that are + too slow for normal use. Values are 0 (off), + 1 (on major collections) or 2 (also on minor + collections). """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -36,7 +48,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage from pypy.rpython.memory.gc.base import GCBase, MovingGCBase -from pypy.rpython.memory.gc import minimarkpage, base, generation +from pypy.rpython.memory.gc import minimarkpage, env from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint from pypy.rlib.rarithmetic import LONG_BIT_SHIFT from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop @@ -96,6 +108,7 @@ needs_write_barrier = True prebuilt_gc_objects_are_static_roots = False malloc_zero_filled = True # xxx experiment with False + gcflag_extra = GCFLAG_FINALIZATION_ORDERING # All objects start with a HDR, i.e. with a field 'tid' which contains # a word. This word is divided in two halves: the lower half contains @@ -154,7 +167,7 @@ # grow at most by the following factor from one collection to the # next. Used e.g. when there is a sudden, temporary peak in memory # usage; this avoids that the upper bound grows too fast. - "growth_rate_max": 1.3, + "growth_rate_max": 1.4, # The number of array indices that are mapped to a single bit in # write_barrier_from_array(). Must be a power of two. The default @@ -198,6 +211,7 @@ self.min_heap_size = 0.0 self.max_heap_size = 0.0 self.max_heap_size_already_raised = False + self.max_delta = float(r_uint(-1)) # self.card_page_indices = card_page_indices if self.card_page_indices > 0: @@ -215,7 +229,8 @@ self.nursery = NULL self.nursery_free = NULL self.nursery_top = NULL - self.debug_always_do_minor_collect = False + self.debug_tiny_nursery = -1 + self.debug_rotating_nurseries = None # # The ArenaCollection() handles the nonmovable objects allocation. if ArenaCollectionClass is None: @@ -282,53 +297,71 @@ # # From there on, the GC is fully initialized and the code # below can use it - newsize = base.read_from_env('PYPY_GC_NURSERY') - # PYPY_GC_NURSERY=1 forces a minor collect for every malloc. - # Useful to debug external factors, like trackgcroot or the - # handling of the write barrier. - self.debug_always_do_minor_collect = newsize == 1 + newsize = env.read_from_env('PYPY_GC_NURSERY') + # PYPY_GC_NURSERY=smallvalue means that minor collects occur + # very frequently; the extreme case is PYPY_GC_NURSERY=1, which + # forces a minor collect for every malloc. Useful to debug + # external factors, like trackgcroot or the handling of the write + # barrier. Implemented by still using 'minsize' for the nursery + # size (needed to handle e.g. mallocs of 8249 words) but hacking + # at the current nursery position in collect_and_reserve(). if newsize <= 0: - newsize = generation.estimate_best_nursery_size() + newsize = env.estimate_best_nursery_size() if newsize <= 0: newsize = defaultsize - newsize = max(newsize, minsize) + if newsize < minsize: + self.debug_tiny_nursery = newsize & ~(WORD-1) + newsize = minsize # - major_coll = base.read_float_from_env('PYPY_GC_MAJOR_COLLECT') + major_coll = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT') if major_coll > 1.0: self.major_collection_threshold = major_coll # - growth = base.read_float_from_env('PYPY_GC_GROWTH') + growth = env.read_float_from_env('PYPY_GC_GROWTH') if growth > 1.0: self.growth_rate_max = growth # - min_heap_size = base.read_uint_from_env('PYPY_GC_MIN') + min_heap_size = env.read_uint_from_env('PYPY_GC_MIN') if min_heap_size > 0: self.min_heap_size = float(min_heap_size) else: # defaults to 8 times the nursery self.min_heap_size = newsize * 8 # - max_heap_size = base.read_uint_from_env('PYPY_GC_MAX') + max_heap_size = env.read_uint_from_env('PYPY_GC_MAX') if max_heap_size > 0: self.max_heap_size = float(max_heap_size) # + max_delta = env.read_uint_from_env('PYPY_GC_MAX_DELTA') + if max_delta > 0: + self.max_delta = float(max_delta) + else: + self.max_delta = 0.125 * env.get_total_memory() + # self.minor_collection() # to empty the nursery llarena.arena_free(self.nursery) self.nursery_size = newsize self.allocate_nursery() - def allocate_nursery(self): - debug_start("gc-set-nursery-size") - debug_print("nursery size:", self.nursery_size) + def _nursery_memory_size(self): + extra = self.nonlarge_gcptrs_max + 1 + return self.nursery_size + extra + + def _alloc_nursery(self): # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize_clear(). The few extra pages are never used # anyway so it doesn't even count. - extra = self.nonlarge_gcptrs_max + 1 - self.nursery = llarena.arena_malloc(self.nursery_size + extra, 2) - if not self.nursery: + nursery = llarena.arena_malloc(self._nursery_memory_size(), 2) + if not nursery: raise MemoryError("cannot allocate nursery") + return nursery + + def allocate_nursery(self): + debug_start("gc-set-nursery-size") + debug_print("nursery size:", self.nursery_size) + self.nursery = self._alloc_nursery() # the current position in the nursery: self.nursery_free = self.nursery # the end of the nursery: @@ -362,6 +395,40 @@ return bounded + def post_setup(self): + # set up extra stuff for PYPY_GC_DEBUG. + MovingGCBase.post_setup(self) + if self.DEBUG and llarena.has_protect: + # gc debug mode: allocate 23 nurseries instead of just 1, + # and use them alternatively, while mprotect()ing the unused + # ones to detect invalid access. + debug_start("gc-debug") + self.debug_rotating_nurseries = [] + for i in range(22): + nurs = self._alloc_nursery() + llarena.arena_protect(nurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(nurs) + debug_print("allocated", len(self.debug_rotating_nurseries), + "extra nurseries") + debug_stop("gc-debug") + + def debug_rotate_nursery(self): + if self.debug_rotating_nurseries is not None: + debug_start("gc-debug") + oldnurs = self.nursery + llarena.arena_protect(oldnurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(oldnurs) + # + newnurs = self.debug_rotating_nurseries.pop(0) + llarena.arena_protect(newnurs, self._nursery_memory_size(), False) + self.nursery = newnurs + self.nursery_top = self.nursery + self.nursery_size + debug_print("switching from nursery", oldnurs, + "to nursery", self.nursery, + "size", self.nursery_size) + debug_stop("gc-debug") + + def malloc_fixedsize_clear(self, typeid, size, can_collect=True, needs_finalizer=False, contains_weakptr=False): ll_assert(can_collect, "!can_collect") @@ -499,8 +566,9 @@ self.nursery_free = result + totalsize ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow") # - if self.debug_always_do_minor_collect: - self.nursery_free = self.nursery_top + if self.debug_tiny_nursery >= 0: # for debugging + if self.nursery_top - self.nursery_free > self.debug_tiny_nursery: + self.nursery_free = self.nursery_top - self.debug_tiny_nursery # return result collect_and_reserve._dont_inline_ = True @@ -638,8 +706,13 @@ # means recording that they have a smaller size, so that when # moved out of the nursery, they will consume less memory. # In particular, an array with GCFLAG_HAS_CARDS is never resized. + # Also, a nursery object with GCFLAG_HAS_SHADOW is not resized + # either, as this would potentially loose part of the memory in + # the already-allocated shadow. if not self.is_in_nursery(obj): return False + if self.header(obj).tid & GCFLAG_HAS_SHADOW: + return False # size_gc_header = self.gcheaderbuilder.size_gc_header typeid = self.get_type_id(obj) @@ -822,7 +895,7 @@ def remember_young_pointer(addr_struct, newvalue): # 'addr_struct' is the address of the object in which we write. # 'newvalue' is the address that we are going to write in there. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") # @@ -859,7 +932,7 @@ # 'addr_array' is the address of the object in which we write, # which must have an array part; 'index' is the index of the # item that is (or contains) the pointer that we write. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_array), "nursery array with GCFLAG_NO_YOUNG_PTRS") objhdr = self.header(addr_array) @@ -981,13 +1054,14 @@ # All live nursery objects are out, and the rest dies. Fill # the whole nursery with zero and reset the current nursery pointer. llarena.arena_reset(self.nursery, self.nursery_size, 2) + self.debug_rotate_nursery() self.nursery_free = self.nursery # debug_print("minor collect, total memory used:", self.get_total_memory_used()) + if self.DEBUG >= 2: + self.debug_check_consistency() # expensive! debug_stop("gc-minor") - if 0: # not we_are_translated(): - self.debug_check_consistency() # xxx expensive! def collect_roots_in_nursery(self): @@ -1205,10 +1279,6 @@ self.collect_roots() self.visit_all_objects() # - # Weakref support: clear the weak pointers to dying objects - if self.old_objects_with_weakrefs.non_empty(): - self.invalidate_old_weakrefs() - # # Finalizer support: adds the flag GCFLAG_VISITED to all objects # with a finalizer and all objects reachable from there (and also # moves some objects from 'objects_with_finalizers' to @@ -1218,6 +1288,10 @@ # self.objects_to_trace.delete() # + # Weakref support: clear the weak pointers to dying objects + if self.old_objects_with_weakrefs.non_empty(): + self.invalidate_old_weakrefs() + # # Walk all rawmalloced objects and free the ones that don't # have the GCFLAG_VISITED flag. self.free_unvisited_rawmalloc_objects() @@ -1245,9 +1319,12 @@ # # Set the threshold for the next major collection to be when we # have allocated 'major_collection_threshold' times more than + # we currently have -- but no more than 'max_delta' more than # we currently have. + total_memory_used = float(self.get_total_memory_used()) bounded = self.set_major_threshold_from( - self.get_total_memory_used() * self.major_collection_threshold, + min(total_memory_used * self.major_collection_threshold, + total_memory_used + self.max_delta), reserving_size) # # Max heap size: gives an upper bound on the threshold. If we @@ -1406,12 +1483,21 @@ size = self.get_size(obj) shadowhdr = self._malloc_out_of_nursery(size_gc_header + size) - # initialize to an invalid tid *without* GCFLAG_VISITED, - # so that if the object dies before the next minor - # collection, the shadow will stay around but be collected - # by the next major collection. + # Initialize the shadow enough to be considered a + # valid gc object. If the original object stays + # alive at the next minor collection, it will anyway + # be copied over the shadow and overwrite the + # following fields. But if the object dies, then + # the shadow will stay around and only be freed at + # the next major collection, at which point we want + # it to look valid (but ready to be freed). shadow = shadowhdr + size_gc_header - self.header(shadow).tid = 0 + self.header(shadow).tid = self.header(obj).tid + typeid = self.get_type_id(obj) + if self.is_varsize(typeid): + lenofs = self.varsize_offset_to_length(typeid) + (shadow + lenofs).signed[0] = (obj + lenofs).signed[0] + # self.header(obj).tid |= GCFLAG_HAS_SHADOW self.young_objects_shadows.setitem(obj, shadow) # Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/semispace.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/semispace.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/semispace.py Sat Dec 11 15:10:15 2010 @@ -42,6 +42,7 @@ inline_simple_malloc_varsize = True malloc_zero_filled = True first_unused_gcflag = first_gcflag << 5 + gcflag_extra = GCFLAG_FINALIZATION_ORDERING HDR = lltype.Struct('header', ('tid', lltype.Signed)) # XXX or rffi.INT? typeid_is_in_field = 'tid' @@ -266,10 +267,10 @@ if self.run_finalizers.non_empty(): self.update_run_finalizers() scan = self.scan_copied(scan) - if self.objects_with_weakrefs.non_empty(): - self.invalidate_weakrefs() if self.objects_with_finalizers.non_empty(): scan = self.deal_with_objects_with_finalizers(scan) + if self.objects_with_weakrefs.non_empty(): + self.invalidate_weakrefs() self.update_objects_with_id() self.finished_full_collect() self.debug_check_consistency() Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/test/test_direct.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/test/test_direct.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gc/test/test_direct.py Sat Dec 11 15:10:15 2010 @@ -60,7 +60,7 @@ pass -class DirectGCTest(object): +class BaseDirectGCTest(object): GC_PARAMS = {} def setup_method(self, meth): @@ -106,6 +106,9 @@ addr = self.gc.malloc(self.get_type_id(TYPE), n, zero=True) return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE)) + +class DirectGCTest(BaseDirectGCTest): + def test_simple(self): p = self.malloc(S) p.x = 5 @@ -339,6 +342,15 @@ self.gc.collect() assert hash == self.gc.identityhash(self.stackroots[-1]) self.stackroots.pop() + # (7) the same, but the objects are dying young + for i in range(10): + self.gc.collect() + p = self.malloc(VAR, i) + self.stackroots.append(p) + hash1 = self.gc.identityhash(p) + hash2 = self.gc.identityhash(p) + assert hash1 == hash2 + self.stackroots.pop() def test_memory_alignment(self): A1 = lltype.GcArray(lltype.Char) Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/asmgcroot.py Sat Dec 11 15:10:15 2010 @@ -139,12 +139,13 @@ self._shape_decompressor = ShapeDecompressor() if hasattr(gctransformer.translator, '_jit2gc'): jit2gc = gctransformer.translator._jit2gc - self._extra_gcmapstart = jit2gc['gcmapstart'] - self._extra_gcmapend = jit2gc['gcmapend'] + self._extra_gcmapstart = jit2gc['gcmapstart'] + self._extra_gcmapend = jit2gc['gcmapend'] + self._extra_mark_sorted = jit2gc['gcmarksorted'] else: - returns_null = lambda: llmemory.NULL - self._extra_gcmapstart = returns_null - self._extra_gcmapend = returns_null + self._extra_gcmapstart = lambda: llmemory.NULL + self._extra_gcmapend = lambda: llmemory.NULL + self._extra_mark_sorted = lambda: True def need_thread_support(self, gctransformer, getfn): # Threads supported "out of the box" by the rest of the code. @@ -295,13 +296,25 @@ # we have a non-empty JIT-produced table to look in item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) if item: - self._shape_decompressor.setaddr(item.address[1]) + self._shape_decompressor.setaddr(item) return # maybe the JIT-produced table is not sorted? + was_already_sorted = self._extra_mark_sorted() + if not was_already_sorted: + sort_gcmap(gcmapstart2, gcmapend2) + item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) + if item: + self._shape_decompressor.setaddr(item) + return + # there is a rare risk that the array contains *two* entries + # with the same key, one of which is dead (null value), and we + # found the dead one above. Solve this case by replacing all + # dead keys with nulls, sorting again, and then trying again. + replace_dead_entries_with_nulls(gcmapstart2, gcmapend2) sort_gcmap(gcmapstart2, gcmapend2) item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) if item: - self._shape_decompressor.setaddr(item.address[1]) + self._shape_decompressor.setaddr(item) return # the item may have been not found because the main array was # not sorted. Sort it and try again. @@ -357,7 +370,8 @@ The interval from the start address (included) to the end address (excluded) is assumed to be a sorted arrays of pairs (addr1, addr2). This searches for the item with a given addr1 and returns its - address. + address. If not found exactly, it tries to return the address + of the item left of addr1 (i.e. such that result.address[0] < addr1). """ count = (end - start) // arrayitemsize while count > 1: @@ -386,7 +400,7 @@ # (item.signed[1] is an address in this case, not a signed at all!) item = binary_search(gcmapstart, gcmapend, retaddr) if item.address[0] == retaddr: - return item # found + return item.address[1] # found else: return llmemory.NULL # failed @@ -397,6 +411,15 @@ rffi.cast(rffi.SIZE_T, arrayitemsize), llhelper(QSORT_CALLBACK_PTR, _compare_gcmap_entries)) +def replace_dead_entries_with_nulls(start, end): + # replace the dead entries (null value) with a null key. + count = (end - start) // arrayitemsize - 1 + while count >= 0: + item = start + count * arrayitemsize + if item.address[1] == llmemory.NULL: + item.address[0] = llmemory.NULL + count -= 1 + if sys.platform == 'win32': def win32_follow_gcmap_jmp(start, end): # The initial gcmap table contains addresses to a JMP Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/boehm.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/boehm.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/boehm.py Sat Dec 11 15:10:15 2010 @@ -120,10 +120,11 @@ fptr = self.annotate_finalizer(d['ll_finalizer'], [llmemory.Address], lltype.Void) elif destrptr: EXC_INSTANCE_TYPE = self.translator.rtyper.exceptiondata.lltype_of_exception_value + typename = TYPE.__name__ def ll_finalizer(addr): exc_instance = llop.gc_fetch_exception(EXC_INSTANCE_TYPE) v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) llop.gc_restore_exception(lltype.Void, exc_instance) fptr = self.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) else: Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/framework.py Sat Dec 11 15:10:15 2010 @@ -189,6 +189,7 @@ # run-time initialization code root_walker.setup_root_walker() gcdata.gc.setup() + gcdata.gc.post_setup() def frameworkgc__teardown(): # run-time teardown code for tests! @@ -1204,9 +1205,10 @@ assert not type_contains_pyobjs(TYPE), "not implemented" if destrptr: + typename = TYPE.__name__ def ll_finalizer(addr): v = llmemory.cast_adr_to_ptr(addr, DESTR_ARG) - ll_call_destructor(destrptr, v) + ll_call_destructor(destrptr, v, typename) fptr = self.transformer.annotate_finalizer(ll_finalizer, [llmemory.Address], lltype.Void) Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/refcounting.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/refcounting.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/refcounting.py Sat Dec 11 15:10:15 2010 @@ -227,7 +227,7 @@ # refcount is at zero, temporarily bump it to 1: gcheader.refcount = 1 destr_v = cast_pointer(DESTR_ARG, v) - ll_call_destructor(destrptr, destr_v) + ll_call_destructor(destrptr, destr_v, %r) refcount = gcheader.refcount - 1 gcheader.refcount = refcount if refcount == 0: @@ -239,7 +239,7 @@ pop_alive(exc_instance) # XXX layering of exceptiontransform versus gcpolicy -""" % (body, TYPE._gckind) +""" % (TYPE.__name__, body, TYPE._gckind) else: call_del = None body = '\n'.join(_static_deallocator_body_for_type('v', TYPE)) Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/support.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/support.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/gctransform/support.py Sat Dec 11 15:10:15 2010 @@ -98,11 +98,15 @@ hop.exception_cannot_occur() return hop.inputconst(hop.r_result.lowleveltype, hop.s_result.const) -def ll_call_destructor(destrptr, destr_v): +def ll_call_destructor(destrptr, destr_v, typename): try: destrptr(destr_v) - except: + except Exception, e: try: - os.write(2, "a destructor raised an exception, ignoring it\n") + os.write(2, "a destructor of type ") + os.write(2, typename) + os.write(2, " raised an exception ") + os.write(2, str(e)) + os.write(2, " ignoring it\n") except: pass Modified: pypy/branch/jit-unroll-loops/pypy/rpython/memory/test/test_gc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/memory/test/test_gc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/memory/test/test_gc.py Sat Dec 11 15:10:15 2010 @@ -304,7 +304,7 @@ a = A() class B(object): def __del__(self): - # when __del__ is called, the weakref should have been cleared + # when __del__ is called, the weakref to c should be dead if self.ref() is None: a.count += 10 # ok else: @@ -333,8 +333,10 @@ a = A() class B(object): def __del__(self): - # when __del__ is called, the weakref should have been cleared - if self.ref() is None: + # when __del__ is called, the weakref to myself is still valid + # in RPython (at least with most GCs; this test might be + # skipped for specific GCs) + if self.ref() is self: a.count += 10 # ok else: a.count = 666 # not ok @@ -352,6 +354,29 @@ res = self.interpret(f, []) assert res == 11 + def test_weakref_bug_1(self): + import weakref + class A(object): + pass + class B(object): + def __del__(self): + self.wref().x += 1 + def g(a): + b = B() + b.wref = weakref.ref(a) + # the only way to reach this weakref is via B, which is an + # object with finalizer (but the weakref itself points to + # a, which does not go away but will move during the next + # gc.collect) + def f(): + a = A() + a.x = 10 + g(a) + llop.gc__collect(lltype.Void) + return a.x + res = self.interpret(f, []) + assert res == 11 + def test_id(self): class A(object): pass @@ -709,6 +734,9 @@ class TestMarkSweepGC(GCTest): from pypy.rpython.memory.gc.marksweep import MarkSweepGC as GCClass + def test_weakref_to_object_with_finalizer_ordering(self): + py.test.skip("Does not work") + class TestSemiSpaceGC(GCTest, snippet.SemiSpaceGCTests): from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass GC_CAN_MOVE = True @@ -731,9 +759,6 @@ def test_finalizer_order(self): py.test.skip("Not implemented yet") - def test_weakref_to_object_with_finalizer_ordering(self): - py.test.skip("Not implemented yet") - class TestHybridGC(TestGenerationalGC): from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass GC_CAN_MALLOC_NONMOVABLE = True Modified: pypy/branch/jit-unroll-loops/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/module/ll_os.py Sat Dec 11 15:10:15 2010 @@ -730,6 +730,22 @@ return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) + @registering_if(os, 'getloadavg') + def register_os_getloadavg(self): + AP = rffi.CArrayPtr(lltype.Float) + c_getloadavg = self.llexternal('getloadavg', [AP, rffi.INT], rffi.INT) + + def getloadavg_llimpl(): + load = lltype.malloc(AP.TO, 3, flavor='raw') + r = c_getloadavg(load, 3) + result_tuple = load[0], load[1], load[2] + lltype.free(load, flavor='raw') + if r != 3: + raise OSError + return result_tuple + return extdef([], (float, float, float), + "ll_os.ll_getloadavg", llimpl=getloadavg_llimpl) + # ------------------------------- os.read ------------------------------- @registering(os.read) @@ -887,6 +903,18 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") + @registering_if(os, 'fchdir') + def register_os_fchdir(self): + os_fchdir = self.llexternal('fchdir', [rffi.INT], rffi.INT) + + def fchdir_llimpl(fd): + res = rffi.cast(rffi.LONG, os_fchdir(rffi.cast(rffi.INT, fd))) + if res < 0: + raise OSError(rposix.get_errno(), "fchdir failed") + return extdef([int], s_None, + llimpl=fchdir_llimpl, + export_name="ll_os.ll_os_fchdir") + @registering_str_unicode(os.access) def register_os_access(self, traits): os_access = self.llexternal(traits.posix_function_name('access'), @@ -1386,6 +1414,25 @@ return extdef([], (int, int), "ll_os.ll_os_openpty", llimpl=openpty_llimpl) + @registering_if(os, 'forkpty') + def register_os_forkpty(self): + os_forkpty = self.llexternal( + 'forkpty', + [rffi.INTP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP], + rffi.PID_T, + compilation_info=ExternalCompilationInfo(libraries=['util'])) + def forkpty_llimpl(): + master_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw') + childpid = os_forkpty(master_p, None, None, None) + master_fd = master_p[0] + lltype.free(master_p, flavor='raw') + if childpid == -1: + raise OSError(rposix.get_errno(), "os_forkpty failed") + return (rffi.cast(lltype.Signed, childpid), + rffi.cast(lltype.Signed, master_fd)) + + return extdef([], (int, int), "ll_os.ll_os_forkpty", + llimpl=forkpty_llimpl) @registering(os._exit) def register_os__exit(self): Modified: pypy/branch/jit-unroll-loops/pypy/rpython/module/ll_time.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/module/ll_time.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/module/ll_time.py Sat Dec 11 15:10:15 2010 @@ -41,7 +41,7 @@ RUSAGE = platform.Struct('struct rusage', [('ru_utime', TIMEVAL), ('ru_stime', TIMEVAL)]) -if sys.platform == 'freebsd7': +if "freebsd" in sys.platform: libraries = ['compat'] else: libraries = [] Modified: pypy/branch/jit-unroll-loops/pypy/rpython/ootypesystem/rpbc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/ootypesystem/rpbc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/ootypesystem/rpbc.py Sat Dec 11 15:10:15 2010 @@ -49,7 +49,7 @@ return hop.genop('runtimenew', [v_class], resulttype=resulttype) def getlowleveltype(self): - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) # if any of the classdefs get the lowleveltype ootype.Class, # we can only pick ootype.Class for us too. Otherwise META. for classdesc in classdescs: @@ -70,7 +70,7 @@ class MethodImplementations(object): def __init__(self, rtyper, methdescs): - samplemdesc = methdescs.iterkeys().next() + samplemdesc = iter(methdescs).next() concretetable, uniquerows = get_concrete_calltable(rtyper, samplemdesc.funcdesc.getcallfamily()) self.row_mapping = {} @@ -117,7 +117,7 @@ concretetable = None # set by _setup_repr_final def _setup_repr_final(self): - sampledesc = self.s_pbc.descriptions.iterkeys().next() + sampledesc = self.s_pbc.any_description() self.concretetable, _ = get_concrete_calltable(self.rtyper, sampledesc.funcdesc.getcallfamily()) Modified: pypy/branch/jit-unroll-loops/pypy/rpython/rlist.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/rlist.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/rlist.py Sat Dec 11 15:10:15 2010 @@ -9,7 +9,7 @@ from pypy.rpython import robject from pypy.rlib.objectmodel import malloc_zero_filled from pypy.rlib.debug import ll_assert -from pypy.rlib.rarithmetic import ovfcheck, widen +from pypy.rlib.rarithmetic import ovfcheck, widen, r_uint, intmask from pypy.rpython.annlowlevel import ADTInterface from pypy.rlib import rgc @@ -241,17 +241,22 @@ class __extend__(pairtype(AbstractBaseListRepr, IntegerRepr)): def rtype_getitem((r_lst, r_int), hop, checkidx=False): + v_lst, v_index = hop.inputargs(r_lst, Signed) if checkidx: - spec = dum_checkidx + hop.exception_is_here() else: - spec = dum_nocheck - v_func = hop.inputconst(Void, spec) - v_lst, v_index = hop.inputargs(r_lst, Signed) + hop.exception_cannot_occur() if hop.args_s[0].listdef.listitem.mutated or checkidx: if hop.args_s[1].nonneg: llfn = ll_getitem_nonneg else: llfn = ll_getitem + if checkidx: + spec = dum_checkidx + else: + spec = dum_nocheck + c_func_marker = hop.inputconst(Void, spec) + v_res = hop.gendirectcall(llfn, c_func_marker, v_lst, v_index) else: # this is the 'foldable' version, which is not used when # we check for IndexError @@ -259,11 +264,7 @@ llfn = ll_getitem_foldable_nonneg else: llfn = ll_getitem_foldable - if checkidx: - hop.exception_is_here() - else: - hop.exception_cannot_occur() - v_res = hop.gendirectcall(llfn, v_func, v_lst, v_index) + v_res = hop.gendirectcall(llfn, v_lst, v_index) return r_lst.recast(hop.llops, v_res) rtype_getitem_key = rtype_getitem @@ -538,12 +539,14 @@ dest.ll_setitem_fast(dest_start + i, item) i += 1 ll_arraycopy._annenforceargs_ = [None, None, int, int, int] +# no oopspec -- the function is inlined by the JIT def ll_copy(RESLIST, l): length = l.ll_length() new_lst = RESLIST.ll_newlist(length) ll_arraycopy(l, new_lst, 0, 0, length) return new_lst +# no oopspec -- the function is inlined by the JIT def ll_len(l): return l.ll_length() @@ -551,6 +554,7 @@ def ll_list_is_true(l): # check if a list is True, allowing for None return bool(l) and l.ll_length() != 0 +# no oopspec -- the function is inlined by the JIT def ll_len_foldable(l): return l.ll_length() @@ -558,6 +562,7 @@ def ll_list_is_true_foldable(l): return bool(l) and ll_len_foldable(l) != 0 +# no oopspec -- the function is inlined by the JIT def ll_append(l, newitem): length = l.ll_length() @@ -588,6 +593,7 @@ ll_arraycopy(l1, l, 0, 0, len1) ll_arraycopy(l2, l, 0, len1, len2) return l +# no oopspec -- the function is inlined by the JIT def ll_insert_nonneg(l, index, newitem): length = l.ll_length() @@ -674,60 +680,72 @@ l.ll_setitem_fast(length_1_i, tmp) i += 1 length_1_i -= 1 +ll_reverse.oopspec = 'list.reverse(l)' def ll_getitem_nonneg(func, l, index): ll_assert(index >= 0, "unexpectedly negative list getitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError - else: - ll_assert(index < l.ll_length(), "list getitem index out of bound") return l.ll_getitem_fast(index) -ll_getitem_nonneg.oopspec = 'list.getitem(l, index)' +ll_getitem_nonneg._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_getitem(func, l, index): - length = l.ll_length() - if index < 0: - index += length if func is dum_checkidx: - if index < 0 or index >= length: - raise IndexError + length = l.ll_length() # common case: 0 <= index < length + if r_uint(index) >= r_uint(length): + # Failed, so either (-length <= index < 0), or we have to raise + # IndexError. First add 'length' to get the final index, then + # check that we now have (0 <= index < length). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(index >= 0, "negative list getitem index out of bound") - ll_assert(index < length, "list getitem index out of bound") + # We don't want checking, but still want to support index < 0. + # Only call ll_length() if needed. + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list getitem index out of bound") return l.ll_getitem_fast(index) -ll_getitem.oopspec = 'list.getitem(l, index)' +# no oopspec -- the function is inlined by the JIT -def ll_getitem_foldable_nonneg(func, l, index): - return ll_getitem_nonneg(func, l, index) +def ll_getitem_foldable_nonneg(l, index): + ll_assert(index >= 0, "unexpectedly negative list getitem index") + return l.ll_getitem_fast(index) ll_getitem_foldable_nonneg.oopspec = 'list.getitem_foldable(l, index)' -def ll_getitem_foldable(func, l, index): - return ll_getitem(func, l, index) -ll_getitem_foldable.oopspec = 'list.getitem_foldable(l, index)' +def ll_getitem_foldable(l, index): + if index < 0: + index += l.ll_length() + return ll_getitem_foldable_nonneg(l, index) +ll_getitem_foldable._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_setitem_nonneg(func, l, index, newitem): ll_assert(index >= 0, "unexpectedly negative list setitem index") if func is dum_checkidx: if index >= l.ll_length(): raise IndexError - else: - ll_assert(index < l.ll_length(), "list setitem index out of bound") l.ll_setitem_fast(index, newitem) -ll_setitem_nonneg.oopspec = 'list.setitem(l, index, newitem)' +ll_setitem_nonneg._always_inline_ = True +# no oopspec -- the function is inlined by the JIT def ll_setitem(func, l, index, newitem): - length = l.ll_length() - if index < 0: - index += length if func is dum_checkidx: - if index < 0 or index >= length: - raise IndexError + length = l.ll_length() + if r_uint(index) >= r_uint(length): # see comments in ll_getitem(). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(index >= 0, "negative list setitem index out of bound") - ll_assert(index < length, "list setitem index out of bound") + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list setitem index out of bound") l.ll_setitem_fast(index, newitem) -ll_setitem.oopspec = 'list.setitem(l, index, newitem)' +# no oopspec -- the function is inlined by the JIT def ll_delitem_nonneg(func, l, index): ll_assert(index >= 0, "unexpectedly negative list delitem index") @@ -751,19 +769,20 @@ l._ll_resize_le(newlength) ll_delitem_nonneg.oopspec = 'list.delitem(l, index)' -def ll_delitem(func, l, i): - length = l.ll_length() - if i < 0: - i += length +def ll_delitem(func, l, index): if func is dum_checkidx: - if i < 0 or i >= length: - raise IndexError + length = l.ll_length() + if r_uint(index) >= r_uint(length): # see comments in ll_getitem(). + index = r_uint(index) + r_uint(length) + if index >= r_uint(length): + raise IndexError + index = intmask(index) else: - ll_assert(i >= 0, "negative list delitem index out of bound") - ll_assert(i < length, "list delitem index out of bound") - ll_delitem_nonneg(dum_nocheck, l, i) -ll_delitem.oopspec = 'list.delitem(l, i)' - + if index < 0: + index += l.ll_length() + ll_assert(index >= 0, "negative list delitem index out of bound") + ll_delitem_nonneg(dum_nocheck, l, index) +# no oopspec -- the function is inlined by the JIT def ll_extend(l1, l2): len1 = l1.ll_length() @@ -799,6 +818,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_str_slice_startstop(lst, s, getstrlen, getstritem, start, stop): @@ -824,6 +844,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_str_slice_minusone(lst, s, getstrlen, getstritem): len1 = lst.ll_length() @@ -843,6 +864,7 @@ lst.ll_setitem_fast(j, c) i += 1 j += 1 +# not inlined by the JIT -- contains a loop def ll_extend_with_char_count(lst, char, count): if count <= 0: @@ -859,6 +881,7 @@ while j < newlength: lst.ll_setitem_fast(j, char) j += 1 +# not inlined by the JIT -- contains a loop def ll_listslice_startonly(RESLIST, l1, start): len1 = l1.ll_length() @@ -869,6 +892,7 @@ ll_arraycopy(l1, l, start, 0, newlength) return l ll_listslice_startonly._annenforceargs_ = (None, None, int) +# no oopspec -- the function is inlined by the JIT def ll_listslice_startstop(RESLIST, l1, start, stop): length = l1.ll_length() @@ -881,6 +905,7 @@ l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, start, 0, newlength) return l +# no oopspec -- the function is inlined by the JIT def ll_listslice_minusone(RESLIST, l1): newlength = l1.ll_length() - 1 @@ -888,6 +913,7 @@ l = RESLIST.ll_newlist(newlength) ll_arraycopy(l1, l, 0, 0, newlength) return l +# no oopspec -- the function is inlined by the JIT def ll_listdelslice_startonly(l, start): ll_assert(start >= 0, "del l[start:] with unexpectedly negative start") @@ -958,6 +984,7 @@ return False j += 1 return True +# not inlined by the JIT -- contains a loop def ll_listcontains(lst, obj, eqfn): lng = lst.ll_length() @@ -971,6 +998,7 @@ return True j += 1 return False +# not inlined by the JIT -- contains a loop def ll_listindex(lst, obj, eqfn): lng = lst.ll_length() @@ -984,6 +1012,7 @@ return j j += 1 raise ValueError # can't say 'list.index(x): x not in list' +# not inlined by the JIT -- contains a loop def ll_listremove(lst, obj, eqfn): index = ll_listindex(lst, obj, eqfn) # raises ValueError if obj not in lst @@ -1030,3 +1059,4 @@ i += 1 j += length return res +# not inlined by the JIT -- contains a loop Modified: pypy/branch/jit-unroll-loops/pypy/rpython/rpbc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/rpbc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/rpbc.py Sat Dec 11 15:10:15 2010 @@ -15,11 +15,10 @@ from pypy.rpython import callparse - def small_cand(rtyper, s_pbc): if 1 < len(s_pbc.descriptions) < rtyper.getconfig().translation.withsmallfuncsets and \ hasattr(rtyper.type_system.rpbc, 'SmallFunctionSetPBCRepr'): - callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + callfamily = s_pbc.any_description().getcallfamily() concretetable, uniquerows = get_concrete_calltable(rtyper, callfamily) if len(uniquerows) == 1 and (not s_pbc.subset_of or small_cand(rtyper, s_pbc.subset_of)): return True @@ -31,7 +30,7 @@ return none_frozen_pbc_repr kind = self.getKind() if issubclass(kind, description.FunctionDesc): - sample = self.descriptions.keys()[0] + sample = self.any_description() callfamily = sample.querycallfamily() if callfamily and callfamily.total_calltable_size > 0: if sample.overridden: @@ -181,7 +180,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper self.s_pbc = s_pbc - self.callfamily = s_pbc.descriptions.iterkeys().next().getcallfamily() + self.callfamily = s_pbc.any_description().getcallfamily() if len(s_pbc.descriptions) == 1 and not s_pbc.can_be_None: # a single function self.lowleveltype = Void @@ -207,7 +206,7 @@ return self, 0 def get_s_signatures(self, shape): - funcdesc = self.s_pbc.descriptions.iterkeys().next() + funcdesc = self.s_pbc.any_description() return funcdesc.get_s_signatures(shape) ## def function_signatures(self): @@ -322,7 +321,7 @@ bk = self.rtyper.annotator.bookkeeper args = bk.build_args(opname, hop.args_s[1:]) s_pbc = hop.args_s[0] # possibly more precise than self.s_pbc - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) shape, index = description.FunctionDesc.variant_for_call_site(bk, self.callfamily, descs, args) row_of_graphs = self.callfamily.calltables[shape][index] anygraph = row_of_graphs.itervalues().next() # pick any witness @@ -368,7 +367,7 @@ return robject.pyobj_repr def getFrozenPBCRepr(rtyper, s_pbc): - descs = s_pbc.descriptions.keys() + descs = list(s_pbc.descriptions) assert len(descs) >= 1 if len(descs) == 1 and not s_pbc.can_be_None: return SingleFrozenPBCRepr(descs[0]) @@ -530,7 +529,7 @@ def __init__(self, rtyper, s_pbc): self.rtyper = rtyper - self.funcdesc = s_pbc.descriptions.keys()[0].funcdesc + self.funcdesc = s_pbc.any_description().funcdesc # a hack to force the underlying function to show up in call_families # (generally not needed, as normalizecalls() should ensure this, @@ -662,7 +661,7 @@ and the ClassRepr of the class which stores this attribute in its vtable. """ - classdescs = self.s_pbc.descriptions.keys() + classdescs = list(self.s_pbc.descriptions) access = classdescs[0].queryattrfamily(attrname) for classdesc in classdescs[1:]: access1 = classdesc.queryattrfamily(attrname) @@ -819,7 +818,7 @@ if s_pbc.isNone(): raise TyperError("unsupported: variable of type " "bound-method-object or None") - mdescs = s_pbc.descriptions.keys() + mdescs = list(s_pbc.descriptions) methodname = mdescs[0].name classdef = mdescs[0].selfclassdef flags = mdescs[0].flags Modified: pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rclass.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rclass.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rclass.py Sat Dec 11 15:10:15 2010 @@ -917,6 +917,7 @@ assert destrptr is not None def test_del_inheritance(self): + from pypy.rlib import rgc class State: pass s = State() @@ -937,6 +938,7 @@ A() B() C() + rgc.collect() return s.a_dels * 10 + s.b_dels res = f() assert res == 42 @@ -1067,6 +1069,7 @@ assert meth.finalizer def test_del_inheritance(self): + from pypy.rlib import rgc class State: pass s = State() @@ -1087,6 +1090,7 @@ A() B() C() + rgc.collect() return s.a_dels * 10 + s.b_dels res = f() assert res == 42 Modified: pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rint.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rint.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rint.py Sat Dec 11 15:10:15 2010 @@ -4,14 +4,9 @@ from pypy.annotation import model as annmodel from pypy.rpython.test import snippet from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong -from pypy.rlib.rarithmetic import ovfcheck +from pypy.rlib.rarithmetic import ovfcheck, r_int64 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -if r_longlong is not r_int: - int64 = r_longlong -else: - int64 = int - class TestSnippet(object): @@ -110,10 +105,10 @@ def f(i): return str(i) - res = self.interpret(f, [int64(0)]) + res = self.interpret(f, [r_int64(0)]) assert self.ll_to_string(res) == '0' - res = self.interpret(f, [int64(413974738222117)]) + res = self.interpret(f, [r_int64(413974738222117)]) assert self.ll_to_string(res) == '413974738222117' def test_unsigned(self): @@ -135,7 +130,7 @@ f._annspecialcase_ = "specialize:argtype(0)" def g(n): if n > 0: - return f(int64(0)) + return f(r_int64(0)) else: return f(0) res = self.interpret(g, [0]) @@ -147,7 +142,7 @@ def test_downcast_int(self): def f(i): return int(i) - res = self.interpret(f, [int64(0)]) + res = self.interpret(f, [r_int64(0)]) assert res == 0 def test_isinstance_vs_int_types(self): @@ -157,7 +152,7 @@ return [None] if isinstance(x, str): return x - if isinstance(x, int64): + if isinstance(x, r_int64): return int(x) return "XXX" wrap._annspecialcase_ = 'specialize:argtype(0)' @@ -165,7 +160,7 @@ space = FakeSpace() def wrap(x): return space.wrap(x) - res = self.interpret(wrap, [int64(0)]) + res = self.interpret(wrap, [r_int64(0)]) assert res == 0 def test_truediv(self): @@ -178,25 +173,25 @@ def test_float_conversion(self): def f(ii): return float(ii) - res = self.interpret(f, [int64(100000000)]) + res = self.interpret(f, [r_int64(100000000)]) assert type(res) is float assert res == 100000000. - res = self.interpret(f, [int64(1234567890123456789)]) + res = self.interpret(f, [r_int64(1234567890123456789)]) assert type(res) is float assert self.float_eq(res, 1.2345678901234568e+18) def test_float_conversion_implicit(self): def f(ii): return 1.0 + ii - res = self.interpret(f, [int64(100000000)]) + res = self.interpret(f, [r_int64(100000000)]) assert type(res) is float assert res == 100000001. - res = self.interpret(f, [int64(1234567890123456789)]) + res = self.interpret(f, [r_int64(1234567890123456789)]) assert type(res) is float assert self.float_eq(res, 1.2345678901234568e+18) def test_rarithmetic(self): - inttypes = [int, r_uint, int64, r_ulonglong] + inttypes = [int, r_uint, r_int64, r_ulonglong] for inttype in inttypes: c = inttype() def f(): @@ -231,16 +226,16 @@ res = self.interpret(f, [int(-1<<(r_int.BITS-1))]) assert res == 0 - res = self.interpret(f, [int64(-1)]) + res = self.interpret(f, [r_int64(-1)]) assert res == 1 - res = self.interpret(f, [int64(-1)<<(r_longlong.BITS-1)]) + res = self.interpret(f, [r_int64(-1)<<(r_longlong.BITS-1)]) assert res == 0 div_mod_iteration_count = 1000 def test_div_mod(self): import random - for inttype in (int, int64): + for inttype in (int, r_int64): def d(x, y): return x/y @@ -303,7 +298,7 @@ except ZeroDivisionError: return 84 - for inttype in (int, int64): + for inttype in (int, r_int64): args = [( 5, 2), (-5, 2), ( 5,-2), (-5,-2), ( 6, 2), (-6, 2), ( 6,-2), (-6,-2), Modified: pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rlist.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rlist.py (original) +++ pypy/branch/jit-unroll-loops/pypy/rpython/test/test_rlist.py Sat Dec 11 15:10:15 2010 @@ -12,6 +12,7 @@ from pypy.rpython.rint import signed_repr from pypy.objspace.flow.model import Constant, Variable from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin +from pypy.rlib.debug import ll_assert # undo the specialization parameter for n1 in 'get set del'.split(): @@ -1076,7 +1077,13 @@ res = self.interpret(f, [0]) assert res == 1 - py.test.raises(AssertionError, self.interpret, f, [1]) + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) + else: + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def f(x): l = [1] @@ -1121,12 +1128,13 @@ res = self.interpret(f, [0]) assert res == 1 - try: - self.interpret_raises(IndexError, f, [1]) - except (AssertionError,), e: - pass + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) else: - assert False + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def f(x): l = [1] @@ -1163,12 +1171,13 @@ res = self.interpret(f, [0]) assert res == 1 - try: - self.interpret_raises(IndexError, f, [1]) - except (AssertionError,), e: - pass + if self.type_system == 'lltype': + # on lltype we always get an AssertionError + py.test.raises(AssertionError, self.interpret, f, [1]) else: - assert False + # on ootype we happen to get through the ll_asserts and to + # hit the IndexError from ootype.py + self.interpret_raises(IndexError, f, [1]) def test_charlist_extension_1(self): def f(n): @@ -1327,8 +1336,32 @@ res = self.interpret(f, [2]) assert res == True + def test_immutable_list_out_of_instance(self): + from pypy.translator.simplify import get_funcobj + for immutable_fields in (["a", "b"], ["a", "b", "y[*]"]): + class A(object): + _immutable_fields_ = immutable_fields + class B(A): + pass + def f(i): + b = B() + lst = [i] + lst[0] += 1 + b.y = lst + ll_assert(b.y is lst, "copying when reading out the attr?") + return b.y[0] + res = self.interpret(f, [10]) + assert res == 11 + t, rtyper, graph = self.gengraph(f, [int]) + block = graph.startblock + op = block.operations[-1] + assert op.opname == 'direct_call' + func = get_funcobj(op.args[0].value)._callable + assert ('foldable' in func.func_name) == \ + ("y[*]" in immutable_fields) class TestLLtype(BaseTestRlist, LLRtypeMixin): + type_system = 'lltype' rlist = ll_rlist def test_memoryerror(self): @@ -1420,14 +1453,16 @@ lst2 = [i] lst2.append(42) # mutated list return lst1[i] + lst2[i] - _, _, graph = self.gengraph(f, [int]) + from pypy.annotation import model as annmodel + _, _, graph = self.gengraph(f, [annmodel.SomeInteger(nonneg=True)]) block = graph.startblock lst1_getitem_op = block.operations[-3] # XXX graph fishing lst2_getitem_op = block.operations[-2] func1 = lst1_getitem_op.args[0].value._obj._callable func2 = lst2_getitem_op.args[0].value._obj._callable assert func1.oopspec == 'list.getitem_foldable(l, index)' - assert func2.oopspec == 'list.getitem(l, index)' + assert not hasattr(func2, 'oopspec') class TestOOtype(BaseTestRlist, OORtypeMixin): rlist = oo_rlist + type_system = 'ootype' Modified: pypy/branch/jit-unroll-loops/pypy/tool/ansi_print.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/tool/ansi_print.py (original) +++ pypy/branch/jit-unroll-loops/pypy/tool/ansi_print.py Sat Dec 11 15:10:15 2010 @@ -16,6 +16,7 @@ 'WARNING': ((31,), False), 'event': ((1,), True), 'ERROR': ((1, 31), False), + 'Error': ((1, 31), False), 'info': ((35,), False), 'stub': ((34,), False), } Modified: pypy/branch/jit-unroll-loops/pypy/tool/error.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/tool/error.py (original) +++ pypy/branch/jit-unroll-loops/pypy/tool/error.py Sat Dec 11 15:10:15 2010 @@ -120,7 +120,7 @@ msg.append(" (%s getting at the binding!)" % ( e.__class__.__name__,)) return - for desc in descs.keys(): + for desc in list(descs): func = desc.pyobj if func is None: r = repr(desc) Modified: pypy/branch/jit-unroll-loops/pypy/tool/logparser.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/tool/logparser.py (original) +++ pypy/branch/jit-unroll-loops/pypy/tool/logparser.py Sat Dec 11 15:10:15 2010 @@ -12,14 +12,6 @@ from pypy.tool import progressbar def parse_log_file(filename, verbose=True): - r_start = re.compile(r"\[([0-9a-fA-F]+)\] \{([\w-]+)$") - r_stop = re.compile(r"\[([0-9a-fA-F]+)\] ([\w-]+)\}$") - lasttime = 0 - log = DebugLog() - time_decrase = False - performance_log = True - nested = 0 - # f = open(filename, 'r') if f.read(2) == 'BZ': f.close() @@ -30,19 +22,33 @@ lines = f.readlines() f.close() # - if sys.stdout.isatty(): + return parse_log(lines, verbose=verbose) + +def parse_log(lines, verbose=False): + color = "(?:\x1b.*?m)?" + r_start = re.compile(color + r"\[([0-9a-fA-F]+)\] \{([\w-]+)" + color + "$") + r_stop = re.compile(color + r"\[([0-9a-fA-F]+)\] ([\w-]+)\}" + color + "$") + lasttime = 0 + log = DebugLog() + time_decrase = False + performance_log = True + nested = 0 + # + if verbose and sys.stdout.isatty(): progress = progressbar.ProgressBar(color='green') + counter = 0 + else: + progress = None single_percent = len(lines) / 100 if verbose: - vnext = single_percent + vnext = 0 else: - vnext = len(lines) - counter = 0 + vnext = -1 for i, line in enumerate(lines): if i == vnext: - counter += 1 - if sys.stdout.isatty(): + if progress is not None: progress.render(counter) + counter += 1 vnext += single_percent else: sys.stderr.write('%d%%..' % int(100.0*i/len(lines))) Modified: pypy/branch/jit-unroll-loops/pypy/tool/release/force-builds.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/tool/release/force-builds.py (original) +++ pypy/branch/jit-unroll-loops/pypy/tool/release/force-builds.py Sat Dec 11 15:10:15 2010 @@ -20,11 +20,12 @@ 'own-linux-x86-32', 'own-linux-x86-64', # 'own-macosx-x86-32', - 'pypy-c-app-level-linux-x86-32', - 'pypy-c-app-level-linux-x86-64', +# 'pypy-c-app-level-linux-x86-32', +# 'pypy-c-app-level-linux-x86-64', 'pypy-c-stackless-app-level-linux-x86-32', 'pypy-c-app-level-win-x86-32', 'pypy-c-jit-linux-x86-32', + 'pypy-c-jit-linux-x86-64', # 'pypy-c-jit-macosx-x86-32', 'pypy-c-jit-win-x86-32', ] Modified: pypy/branch/jit-unroll-loops/pypy/tool/release/make_release.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/tool/release/make_release.py (original) +++ pypy/branch/jit-unroll-loops/pypy/tool/release/make_release.py Sat Dec 11 15:10:15 2010 @@ -4,7 +4,7 @@ into release packages. Note: you must run apropriate buildbots first and make sure there are no failures. Use force-builds.py from the same directory. -Usage: make_release.py release/ +Usage: make_release.py release/ release_version """ import autopath @@ -30,7 +30,8 @@ else: xml = override_xml dom = minidom.parseString(xml) - refs = [node.getAttribute('href') for node in dom.getElementsByTagName('a')] + refs = [node.getAttribute('href') for node in dom.getElementsByTagName('a') + if 'pypy' in node.getAttribute('href')] # all refs are of form: pypy-{type}-{revision}-{platform}.tar.bz2 r = re.compile('pypy-c-([\w\d]+)-(\d+)-([\w\d]+).tar.bz2$') d = {} @@ -76,7 +77,7 @@ t.add('pypy-%s' % release) alltars.append(name) t.close() - shutil.rmtree(str(tmpdir.join('pypy-1.3'))) + shutil.rmtree(str(tmpdir.join('pypy-' + release))) for name in alltars: print "Uploading %s" % name os.system('scp %s codespeak.net:/www/pypy.org/htdocs/download' % name) @@ -84,8 +85,8 @@ os.chdir(olddir) if __name__ == '__main__': - if len(sys.argv) != 2: + if len(sys.argv) != 3: print __doc__ sys.exit(1) - main(sys.argv[1], release='1.3') + main(sys.argv[1], release=sys.argv[2]) Modified: pypy/branch/jit-unroll-loops/pypy/tool/release/package.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/tool/release/package.py (original) +++ pypy/branch/jit-unroll-loops/pypy/tool/release/package.py Sat Dec 11 15:10:15 2010 @@ -1,8 +1,12 @@ #!/usr/bin/env python """ A sample script that packages PyPy, provided that it's already built. -Usage: +It uses 'pypy/translator/goal/pypy-c' and parts of the rest of the working +copy. Usage: -package.py pypydir [name-of-archive] [name-of-pypy-c] + package.py root-pypy-dir [name-of-archive] [name-of-pypy-c] + +Usually you would do: package.py ../../.. pypy-VER-PLATFORM. +The output is found in the directory /tmp/usession-YOURNAME/build/. """ import autopath @@ -32,7 +36,7 @@ class PyPyCNotFound(Exception): pass -def package(basedir, name='pypy-nightly', rename_pypy_c='pypy-c', +def package(basedir, name='pypy-nightly', rename_pypy_c='pypy', copy_to_dir = None, override_pypy_c = None): basedir = py.path.local(basedir) if sys.platform == 'win32': @@ -64,13 +68,17 @@ headers = includedir.listdir('*.h') + includedir.listdir('*.inl') for n in headers: shutil.copy(str(n), str(pypydir.join('include'))) + # + spdir = pypydir.ensure('site-packages', dir=True) + shutil.copy(str(basedir.join('site-packages', 'README')), str(spdir)) + # pypydir.ensure('bin', dir=True) archive_pypy_c = pypydir.join('bin', rename_pypy_c) shutil.copy(str(pypy_c), str(archive_pypy_c)) old_dir = os.getcwd() try: os.chdir(str(builddir)) - os.system("strip " + str(archive_pypy_c)) # ignore errors + os.system("strip -x " + str(archive_pypy_c)) # ignore errors if USE_TARFILE_MODULE: import tarfile tf = tarfile.open(str(builddir.join(name + '.tar.bz2')), 'w:bz2') Modified: pypy/branch/jit-unroll-loops/pypy/tool/release/test/test_package.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/tool/release/test/test_package.py (original) +++ pypy/branch/jit-unroll-loops/pypy/tool/release/test/test_package.py Sat Dec 11 15:10:15 2010 @@ -18,7 +18,7 @@ prefix = builddir.join(test) cpyver = '%d.%d.%d' % CPYTHON_VERSION[:3] assert prefix.join('lib-python', cpyver, 'test').check() - assert prefix.join('bin', 'pypy-c').check() + assert prefix.join('bin', 'pypy').check() assert prefix.join('lib_pypy', 'syslog.py').check() assert not prefix.join('lib_pypy', 'py').check() assert not prefix.join('lib_pypy', 'ctypes_configure').check() Modified: pypy/branch/jit-unroll-loops/pypy/tool/terminal.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/tool/terminal.py (original) +++ pypy/branch/jit-unroll-loops/pypy/tool/terminal.py Sat Dec 11 15:10:15 2010 @@ -62,9 +62,10 @@ for control in CONTROLS: # Set the control escape sequence setattr(MODULE, control, curses.tigetstr(CONTROLS[control]) or '') - for value in VALUES: - # Set terminal related values - setattr(MODULE, value, curses.tigetnum(VALUES[value])) + if hasattr(curses, 'tigetnum'): + for value in VALUES: + # Set terminal related values + setattr(MODULE, value, curses.tigetnum(VALUES[value])) def render(text): """Helper function to apply controls easily @@ -74,7 +75,16 @@ return text % MODULE.__dict__ try: - import curses + if '__pypy__' in sys.builtin_module_names: + # this is not really the whole curses, but our _minimal_curses it's + # better than nothing + import _minimal_curses as curses + # a bit of a hack: we have tigetstr but not tigetnum, so we call + # default() to have default values, then setup() will overwrite the + # ones it can + default() + else: + import curses setup() except Exception, e: # There is a failure; set all attributes to default Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/funcgen.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/funcgen.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/funcgen.py Sat Dec 11 15:10:15 2010 @@ -1,3 +1,4 @@ +import sys from pypy.translator.c.support import USESLOTS # set to False if necessary while refactoring from pypy.translator.c.support import cdecl from pypy.translator.c.support import llvalue_from_constant, gen_assignments @@ -757,6 +758,16 @@ format.append('%s') argv.append('(%s) ? "True" : "False"' % self.expr(arg)) continue + elif T == SignedLongLong: + if sys.platform == 'win32': + format.append('%I64d') + else: + format.append('%lld') + elif T == UnsignedLongLong: + if sys.platform == 'win32': + format.append('%I64u') + else: + format.append('%llu') else: raise Exception("don't know how to debug_print %r" % (T,)) argv.append(self.expr(arg)) @@ -765,17 +776,20 @@ "if (PYPY_HAVE_DEBUG_PRINTS) { fprintf(PYPY_DEBUG_FILE, %s); %s}" % (', '.join(argv), free_line)) + def _op_debug(self, opname, arg): + if isinstance(arg, Constant): + string_literal = c_string_constant(''.join(arg.value.chars)) + return "%s(%s);" % (opname, string_literal) + else: + x = "%s(RPyString_AsCharP(%s));\n" % (opname, self.expr(arg)) + x += "RPyString_FreeCache();" + return x + def OP_DEBUG_START(self, op): - arg = op.args[0] - assert isinstance(arg, Constant) - return "PYPY_DEBUG_START(%s);" % ( - c_string_constant(''.join(arg.value.chars)),) + return self._op_debug('PYPY_DEBUG_START', op.args[0]) def OP_DEBUG_STOP(self, op): - arg = op.args[0] - assert isinstance(arg, Constant) - return "PYPY_DEBUG_STOP(%s);" % ( - c_string_constant(''.join(arg.value.chars)),) + return self._op_debug('PYPY_DEBUG_STOP', op.args[0]) def OP_DEBUG_ASSERT(self, op): return 'RPyAssert(%s, %s);' % (self.expr(op.args[0]), Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/gcc/test/test_trackgcroot.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/gcc/test/test_trackgcroot.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/gcc/test/test_trackgcroot.py Sat Dec 11 15:10:15 2010 @@ -98,14 +98,13 @@ """ lines = source.splitlines(True) parts = list(DarwinAssemblerParser().find_functions(iter(lines))) - assert len(parts) == 7 + assert len(parts) == 6 assert parts[0] == (False, lines[:3]) assert parts[1] == (True, lines[3:7]) assert parts[2] == (True, lines[7:11]) - assert parts[3] == (True, lines[11:13]) - assert parts[4] == (False, lines[13:18]) - assert parts[5] == (True, lines[18:20]) - assert parts[6] == (False, lines[20:]) + assert parts[3] == (True, lines[11:18]) + assert parts[4] == (True, lines[18:20]) + assert parts[5] == (False, lines[20:]) def test_computegcmaptable(): tests = [] Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/gcc/trackgcroot.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/gcc/trackgcroot.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/gcc/trackgcroot.py Sat Dec 11 15:10:15 2010 @@ -1107,7 +1107,7 @@ format = 'darwin64' function_names_prefix = '_' - LABEL = ElfFunctionGcRootTracker32.LABEL + LABEL = ElfFunctionGcRootTracker64.LABEL r_jmptable_item = re.compile(r"\t.(?:long|quad)\t"+LABEL+"(-\"?[A-Za-z0-9$]+\"?)?\s*$") r_functionstart = re.compile(r"_(\w+):\s*$") @@ -1406,6 +1406,7 @@ 'const_data' ] r_sectionstart = re.compile(r"\t\.("+'|'.join(OTHERSECTIONS)+").*$") + sections_doesnt_end_function = {'cstring': True, 'const': True} def find_functions(self, iterlines): functionlines = [] @@ -1413,20 +1414,20 @@ in_function = False for n, line in enumerate(iterlines): if self.r_textstart.match(line): - assert not in_text, "unexpected repeated .text start: %d" % n in_text = True elif self.r_sectionstart.match(line): - if in_function: + sectionname = self.r_sectionstart.match(line).group(1) + if (in_function and + sectionname not in self.sections_doesnt_end_function): yield in_function, functionlines functionlines = [] + in_function = False in_text = False - in_function = False elif in_text and self.FunctionGcRootTracker.r_functionstart.match(line): yield in_function, functionlines functionlines = [] in_function = True functionlines.append(line) - if functionlines: yield in_function, functionlines @@ -1443,23 +1444,6 @@ format = "mingw32" FunctionGcRootTracker = Mingw32FunctionGcRootTracker - def find_functions(self, iterlines): - functionlines = [] - in_text = False - in_function = False - for n, line in enumerate(iterlines): - if self.r_textstart.match(line): - in_text = True - elif self.r_sectionstart.match(line): - in_text = False - elif in_text and self.FunctionGcRootTracker.r_functionstart.match(line): - yield in_function, functionlines - functionlines = [] - in_function = True - functionlines.append(line) - if functionlines: - yield in_function, functionlines - class MsvcAssemblerParser(AssemblerParser): format = "msvc" FunctionGcRootTracker = MsvcFunctionGcRootTracker Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/src/debug_alloc.h ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/src/debug_alloc.h (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/src/debug_alloc.h Sat Dec 11 15:10:15 2010 @@ -1,5 +1,5 @@ /**************************************************************/ - /*** tracking raw mallocs and frees for debugging ***/ +/*** tracking raw mallocs and frees for debugging ***/ #ifndef RPY_ASSERT @@ -62,8 +62,8 @@ count++; if (count > 0) { - fprintf(stderr, "debug_alloc.h: %ld mallocs left", count); char *env = getenv("PYPY_ALLOC"); + fprintf(stderr, "debug_alloc.h: %ld mallocs left", count); if (env && *env) { fprintf(stderr, " (most recent first):\n"); Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/src/debug_print.h ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/src/debug_print.h (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/src/debug_print.h Sat Dec 11 15:10:15 2010 @@ -10,6 +10,7 @@ but not any nested debug_print :fname full logging prefix:fname conditional logging + prefix1,prefix2:fname conditional logging with multiple selections Conditional logging means that it only includes the debug_start/debug_stop sections whose name match 'prefix'. Other sections are ignored, including @@ -70,6 +71,12 @@ static void pypy_debug_open(void) { char *filename = getenv("PYPYLOG"); + if (filename) +#ifndef MS_WINDOWS + unsetenv("PYPYLOG"); /* don't pass it to subprocesses */ +#else + putenv("PYPYLOG="); /* don't pass it to subprocesses */ +#endif if (filename && filename[0]) { char *colon = strchr(filename, ':'); @@ -139,12 +146,22 @@ #endif -static bool_t startswith(const char *str, const char *substr) +static bool_t startswithoneof(const char *str, const char *substr) { - while (*substr) - if (*str++ != *substr++) - return 0; - return 1; + const char *p = str; + for (; *substr; substr++) + { + if (*substr != ',') + { + if (p && *p++ != *substr) + p = NULL; /* mismatch */ + } + else if (p != NULL) + return 1; /* match */ + else + p = str; /* mismatched, retry with the next */ + } + return p != NULL; } #if defined(_MSC_VER) || defined(__MINGW32__) @@ -175,7 +192,7 @@ if (!debug_profile) { /* non-profiling version */ - if (!debug_prefix || !startswith(category, debug_prefix)) + if (!debug_prefix || !startswithoneof(category, debug_prefix)) { /* wrong section name, or no PYPYLOG at all, skip it */ return; Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/src/g_include.h ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/src/g_include.h (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/src/g_include.h Sat Dec 11 15:10:15 2010 @@ -43,6 +43,10 @@ # include "src/asm_gcc_x86.h" #endif +#if defined(__GNUC__) && defined(__amd64__) +# include "src/asm_gcc_x86_64.h" +#endif + #if defined(__GNUC__) && defined(__ppc__) # include "src/asm_ppc.h" #endif Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/src/mem.h ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/src/mem.h (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/src/mem.h Sat Dec 11 15:10:15 2010 @@ -233,4 +233,4 @@ #define OP_GC_GET_RPY_MEMORY_USAGE(x, r) r = -1 #define OP_GC_GET_RPY_TYPE_INDEX(x, r) r = -1 #define OP_GC_IS_RPY_INSTANCE(x, r) r = 0 -#define OP_GC_DUMP_RPY_HEAP(r) r = 0 +#define OP_GC_DUMP_RPY_HEAP(fd, r) r = 0 Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_extfunc.py Sat Dec 11 15:10:15 2010 @@ -755,3 +755,31 @@ for i in range(5): res = func(i) assert res == os.uname()[i] + +if hasattr(os, 'getloadavg'): + def test_os_getloadavg(): + def does_stuff(): + a, b, c = os.getloadavg() + print a, b, c + return a + b + c + f = compile(does_stuff, []) + res = f() + assert type(res) is float and res >= 0.0 + +if hasattr(os, 'fchdir'): + def test_os_fchdir(): + def does_stuff(): + fd = os.open('/', os.O_RDONLY, 0400) + try: + os.fchdir(fd) + s = os.getcwd() + finally: + os.close(fd) + return s == '/' + f = compile(does_stuff, []) + localdir = os.getcwd() + try: + res = f() + finally: + os.chdir(localdir) + assert res == True Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_newgc.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_newgc.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_newgc.py Sat Dec 11 15:10:15 2010 @@ -1064,14 +1064,16 @@ def test_get_rpy_type_index(self): self.run("get_rpy_type_index") - filename_dump = str(udir.join('test_dump_rpy_heap')) + filename1_dump = str(udir.join('test_dump_rpy_heap.1')) + filename2_dump = str(udir.join('test_dump_rpy_heap.2')) def define_dump_rpy_heap(self): U = lltype.GcForwardReference() U.become(lltype.GcStruct('U', ('next', lltype.Ptr(U)), ('x', lltype.Signed))) S = lltype.GcStruct('S', ('u', lltype.Ptr(U))) A = lltype.GcArray(lltype.Ptr(S)) - filename = self.filename_dump + filename1 = self.filename1_dump + filename2 = self.filename2_dump def fn(): s = lltype.malloc(S) @@ -1081,20 +1083,31 @@ a = lltype.malloc(A, 1000) s2 = lltype.malloc(S) # - fd = os.open(filename, os.O_WRONLY | os.O_CREAT, 0666) - rgc.dump_rpy_heap(fd) + fd1 = os.open(filename1, os.O_WRONLY | os.O_CREAT, 0666) + fd2 = os.open(filename2, os.O_WRONLY | os.O_CREAT, 0666) + rgc.dump_rpy_heap(fd1) + rgc.dump_rpy_heap(fd2) # try twice in a row keepalive_until_here(s2) keepalive_until_here(s) keepalive_until_here(a) - os.close(fd) + os.close(fd1) + os.close(fd2) return 0 return fn def test_dump_rpy_heap(self): self.run("dump_rpy_heap") - assert os.path.exists(self.filename_dump) - assert os.path.getsize(self.filename_dump) > 64 + for fn in [self.filename1_dump, self.filename2_dump]: + assert os.path.exists(fn) + assert os.path.getsize(fn) > 64 + f = open(self.filename1_dump) + data1 = f.read() + f.close() + f = open(self.filename2_dump) + data2 = f.read() + f.close() + assert data1 == data2 filename_dump_typeids_z = str(udir.join('test_typeids_z')) def define_write_typeids_z(self): Modified: pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_standalone.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_standalone.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/c/test/test_standalone.py Sat Dec 11 15:10:15 2010 @@ -272,7 +272,7 @@ x = "got:" debug_start ("mycat") if have_debug_prints(): x += "b" - debug_print ("foo", 2, "bar", 3) + debug_print ("foo", r_longlong(2), "bar", 3) debug_start ("cat2") if have_debug_prints(): x += "c" debug_print ("baz") @@ -368,12 +368,27 @@ assert not err assert path.check(file=1) data = path.read() - assert 'toplevel' in path.read() - assert 'mycat' not in path.read() - assert 'foo 2 bar 3' not in path.read() + assert 'toplevel' in data + assert 'mycat' not in data + assert 'foo 2 bar 3' not in data assert 'cat2' in data assert 'baz' in data assert 'bok' not in data + # check with PYPYLOG=myc,cat2:somefilename (includes mycat and cat2) + path = udir.join('test_debug_xxx_myc_cat2.log') + out, err = cbuilder.cmdexec("", err=True, + env={'PYPYLOG': 'myc,cat2:%s' % path}) + assert out.strip() == 'got:bcda.' + assert not err + assert path.check(file=1) + data = path.read() + assert 'toplevel' in data + assert '{mycat' in data + assert 'mycat}' in data + assert 'foo 2 bar 3' in data + assert 'cat2' in data + assert 'baz' in data + assert 'bok' in data # # finally, check compiling with logging disabled from pypy.config.pypyoption import get_pypy_config @@ -388,6 +403,20 @@ assert not err assert path.check(file=0) + def test_debug_print_start_stop_nonconst(self): + def entry_point(argv): + debug_start(argv[1]) + debug_print(argv[2]) + debug_stop(argv[1]) + return 0 + t, cbuilder = self.compile(entry_point) + out, err = cbuilder.cmdexec("foo bar", err=True, env={'PYPYLOG': ':-'}) + lines = err.splitlines() + assert '{foo' in lines[0] + assert 'bar' == lines[1] + assert 'foo}' in lines[2] + + def test_fatal_error(self): def g(x): if x == 1: Modified: pypy/branch/jit-unroll-loops/pypy/translator/driver.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/driver.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/driver.py Sat Dec 11 15:10:15 2010 @@ -11,7 +11,7 @@ from pypy.annotation import policy as annpolicy import optparse from pypy.tool.udir import udir -from pypy.rlib.jit import DEBUG_OFF, DEBUG_DETAILED, DEBUG_PROFILE, DEBUG_STEPS +from pypy.tool.debug_print import debug_start, debug_print, debug_stop from pypy.rlib.entrypoint import secondary_entrypoints import py @@ -37,13 +37,6 @@ 'c': 'lltype', } -JIT_DEBUG = { - 'off' : DEBUG_OFF, - 'profile' : DEBUG_PROFILE, - 'steps' : DEBUG_STEPS, - 'detailed' : DEBUG_DETAILED, -} - def backend_to_typesystem(backend): return _BACKEND_TO_TYPESYSTEM.get(backend, 'ootype') @@ -283,6 +276,8 @@ return else: self.log.info("%s..." % title) + debug_start('translation-task') + debug_print('starting', goal) self.timer.start_event(goal) try: instrument = False @@ -300,11 +295,13 @@ assert False, 'we should not get here' finally: try: + debug_stop('translation-task') self.timer.end_event(goal) except (KeyboardInterrupt, SystemExit): raise except: pass + #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal) return res def task_annotate(self): @@ -399,7 +396,6 @@ # from pypy.jit.metainterp.warmspot import apply_jit apply_jit(self.translator, policy=self.jitpolicy, - debug_level=JIT_DEBUG[self.config.translation.jit_debug], backend_name=self.config.translation.jit_backend, inline=True) # self.log.info("the JIT compiler was generated") @@ -417,7 +413,6 @@ # from pypy.jit.metainterp.warmspot import apply_jit apply_jit(self.translator, policy=self.jitpolicy, - debug_level=JIT_DEBUG[self.config.translation.jit_debug], backend_name='cli', inline=True) #XXX # self.log.info("the JIT compiler was generated") Modified: pypy/branch/jit-unroll-loops/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/goal/app_main.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/goal/app_main.py Sat Dec 11 15:10:15 2010 @@ -11,6 +11,7 @@ -h, --help show this help message and exit -m library module to be run as a script (terminates option list) -W arg warning control (arg is action:message:category:module:lineno) + -E ignore environment variables (such as PYTHONPATH) --version print the PyPy version --info print translation information about this PyPy executable """ @@ -199,7 +200,7 @@ break # found! return newpath -def setup_initial_paths(executable, nanos): +def setup_initial_paths(executable, nanos, readenv=True, **extra): # a substituted os if we are translated global os os = nanos @@ -220,7 +221,7 @@ sys.executable = os.path.abspath(executable) newpath = get_library_path(executable) - path = os.getenv('PYTHONPATH') + path = readenv and os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath # remove duplicates @@ -230,7 +231,6 @@ if dir not in _seen: sys.path.append(dir) _seen[dir] = True - return executable def parse_command_line(argv): @@ -242,6 +242,7 @@ run_stdin = False warnoptions = [] unbuffered = False + readenv = True while i < len(argv): arg = argv[i] if not arg.startswith('-'): @@ -253,6 +254,8 @@ argv[i] = '-c' run_command = True break + elif arg == '-E': + readenv = False elif arg == '-u': unbuffered = True elif arg == '-O' or arg == '-OO': @@ -305,11 +308,13 @@ run_stdin, warnoptions, unbuffered, + readenv, cmd=None, **ignored): # with PyPy in top of CPython we can only have around 100 - # but we need more in the translated PyPy for the compiler package - sys.setrecursionlimit(5000) + # but we need more in the translated PyPy for the compiler package + if '__pypy__' not in sys.builtin_module_names: + sys.setrecursionlimit(5000) if unbuffered: set_unbuffered_io() @@ -355,7 +360,7 @@ # * PYTHONINSPECT is set and stdin is a tty. # return (go_interactive or - (os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) + (readenv and os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) success = True @@ -388,7 +393,7 @@ # If stdin is a tty or if "-i" is specified, we print # a banner and run $PYTHONSTARTUP. print_banner() - python_startup = os.getenv('PYTHONSTARTUP') + python_startup = readenv and os.getenv('PYTHONSTARTUP') if python_startup: try: f = open(python_startup) @@ -451,7 +456,6 @@ '"license" for more information.') def entry_point(executable, argv, nanos): - executable = setup_initial_paths(executable, nanos) try: cmdline = parse_command_line(argv) except CommandLineError, e: @@ -459,8 +463,8 @@ return 2 if cmdline is None: return 0 - else: - return run_command_line(**cmdline) + setup_initial_paths(executable, nanos, **cmdline) + return run_command_line(**cmdline) if __name__ == '__main__': @@ -495,13 +499,13 @@ sys.pypy_version_info = PYPY_VERSION sys.pypy_initial_path = pypy_initial_path os = nanos.os_module_for_testing - sys.ps1 = '>>>> ' - sys.ps2 = '.... ' try: sys.exit(int(entry_point(sys.argv[0], sys.argv[1:], os))) finally: - sys.ps1 = '>>> ' # restore the normal ones, in case - sys.ps2 = '... ' # we are dropping to CPython's prompt + # restore the normal prompt (which was changed by _pypy_interact), in + # case we are dropping to CPython's prompt + sys.ps1 = '>>> ' + sys.ps2 = '... ' import os; os.environ.update(reset) assert old_argv is sys.argv assert old_path is sys.path Modified: pypy/branch/jit-unroll-loops/pypy/translator/goal/targetpypystandalone.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/goal/targetpypystandalone.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/goal/targetpypystandalone.py Sat Dec 11 15:10:15 2010 @@ -28,9 +28,14 @@ w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish)) w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) w_os = setup_nanos(space) + withjit = space.config.objspace.usemodules.pypyjit def entry_point(argv): space.timer.start("Entrypoint") + if withjit: + from pypy.jit.backend.hlinfo import highleveljitinfo + highleveljitinfo.sys_executable = argv[0] + #debug("entry point starting") #for arg in argv: # debug(" argv -> " + arg) Modified: pypy/branch/jit-unroll-loops/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/goal/test2/test_app_main.py Sat Dec 11 15:10:15 2010 @@ -217,6 +217,38 @@ finally: os.environ['PYTHONSTARTUP'] = old + def test_ignore_python_startup(self): + old = os.environ.get('PYTHONSTARTUP', '') + try: + os.environ['PYTHONSTARTUP'] = crashing_demo_script + child = self.spawn(['-E']) + child.expect(re.escape(banner)) + index = child.expect(['Traceback', '>>> ']) + assert index == 1 # no traceback + finally: + os.environ['PYTHONSTARTUP'] = old + + def test_ignore_python_inspect(self): + os.environ['PYTHONINSPECT_'] = '1' + try: + child = self.spawn(['-E', '-c', 'pass']) + from pexpect import EOF + index = child.expect(['>>> ', EOF]) + assert index == 1 # no prompt + finally: + del os.environ['PYTHONINSPECT_'] + + def test_ignore_python_path(self): + old = os.environ.get('PYTHONPATH', '') + try: + os.environ['PYTHONPATH'] = 'foobarbaz' + child = self.spawn(['-E', '-c', 'import sys; print sys.path']) + from pexpect import EOF + index = child.expect(['foobarbaz', EOF]) + assert index == 1 # no foobarbaz + finally: + os.environ['PYTHONPATH'] = old + def test_unbuffered(self): line = 'import os,sys;sys.stdout.write(str(789));os.read(0,1)' child = self.spawn(['-u', '-c', line]) @@ -329,6 +361,10 @@ child = self.spawn(['-mpypy.translator.goal.test2.mymodule']) child.expect('mymodule running') + def test_ps1_only_if_interactive(self): + argv = ['-c', 'import sys; print hasattr(sys, "ps1")'] + child = self.spawn(argv) + child.expect('False') class TestNonInteractive: @@ -534,7 +570,8 @@ newpath = app_main.get_library_path('/tmp/pypy-c') # stdlib not found assert newpath == sys.path newpath = app_main.get_library_path(self.fake_exe) - assert newpath == self.expected_path + # we get at least 'expected_path', and maybe more (e.g.plat-linux2) + assert newpath[:len(self.expected_path)] == self.expected_path finally: sys.path.pop() @@ -547,7 +584,9 @@ app_main.os = os pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') newpath = app_main.get_library_path(pypy_c) - assert len(newpath) == 3 + # we get at least lib_pypy, lib-python/modified-X.Y.Z, + # lib-python/X.Y.Z, and maybe more (e.g. plat-linux2) + assert len(newpath) >= 3 for p in newpath: assert p.startswith(self.trunkdir) finally: Modified: pypy/branch/jit-unroll-loops/pypy/translator/platform/__init__.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/platform/__init__.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/platform/__init__.py Sat Dec 11 15:10:15 2010 @@ -123,7 +123,9 @@ errorfile.write(stderr, 'wb') stderrlines = stderr.splitlines() for line in stderrlines: - log.ERROR(line) + log.Error(line) + # ^^^ don't use ERROR, because it might actually be fine. + # Also, ERROR confuses lib-python/conftest.py. raise CompilationError(stdout, stderr) else: for line in stderr.splitlines(): @@ -215,13 +217,13 @@ host_factory = Darwin_i386 else: host_factory = Darwin_x86_64 -elif sys.platform == 'freebsd7': - from pypy.translator.platform.freebsd7 import Freebsd7, Freebsd7_64 +elif "freebsd" in sys.platform: + from pypy.translator.platform.freebsd import Freebsd, Freebsd_64 import platform if platform.architecture()[0] == '32bit': - host_factory = Freebsd7 + host_factory = Freebsd else: - host_factory = Freebsd7_64 + host_factory = Freebsd_64 elif os.name == 'nt': from pypy.translator.platform.windows import Windows host_factory = Windows Modified: pypy/branch/jit-unroll-loops/pypy/translator/platform/darwin.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/platform/darwin.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/platform/darwin.py Sat Dec 11 15:10:15 2010 @@ -11,13 +11,17 @@ shared_only = () so_ext = 'so' - + + # NOTE: GCC 4.2 will fail at runtime due to subtle issues, possibly + # related to GC roots. Using LLVM-GCC or Clang will break the build. + default_cc = 'gcc-4.0' + def __init__(self, cc=None): if cc is None: try: cc = os.environ['CC'] except KeyError: - cc = 'gcc' + cc = self.default_cc self.cc = cc def _args_for_shared(self, args): Modified: pypy/branch/jit-unroll-loops/pypy/translator/platform/linux.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/platform/linux.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/platform/linux.py Sat Dec 11 15:10:15 2010 @@ -27,6 +27,8 @@ class Linux(BaseLinux): + shared_only = () # it seems that on 32-bit linux, compiling with -fPIC + # gives assembler that asmgcc is not happy about. def library_dirs_for_libffi_a(self): # places where we need to look for libffi.a return self.library_dirs_for_libffi() + ['/usr/lib'] Modified: pypy/branch/jit-unroll-loops/pypy/translator/platform/posix.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/platform/posix.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/platform/posix.py Sat Dec 11 15:10:15 2010 @@ -104,12 +104,10 @@ else: target_name = exe_name.basename - cflags = self.cflags - if sys.maxint > 2147483647: # XXX XXX XXX sort this out - if shared: - cflags = self.cflags + self.shared_only - else: - cflags = self.cflags + self.standalone_only + if shared: + cflags = self.cflags + self.shared_only + else: + cflags = self.cflags + self.standalone_only m = GnuMakefile(path) m.exe_name = exe_name Modified: pypy/branch/jit-unroll-loops/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/sandbox/test/test_sandbox.py Sat Dec 11 15:10:15 2010 @@ -148,6 +148,7 @@ if sys.platform == 'linux2': # on Mac, uses another (sandboxsafe) approach expect(f, g, "ll_os.ll_os_open", ("/proc/cpuinfo", 0, 420), OSError(5232, "xyz")) + expect(f, g, "ll_os.ll_os_getenv", ("PYPY_GC_DEBUG",), None) g.close() tail = f.read() f.close() Modified: pypy/branch/jit-unroll-loops/pypy/translator/tool/cbuild.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/tool/cbuild.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/tool/cbuild.py Sat Dec 11 15:10:15 2010 @@ -309,6 +309,7 @@ #define _POSIX_C_SOURCE 200112L /* Define on FreeBSD to activate all library features */ #define __BSD_VISIBLE 1 +#define __XSI_VISIBLE 700 /* Windows: winsock/winsock2 mess */ #define WIN32_LEAN_AND_MEAN ''' Modified: pypy/branch/jit-unroll-loops/pypy/translator/tool/reftracker.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/translator/tool/reftracker.py (original) +++ pypy/branch/jit-unroll-loops/pypy/translator/tool/reftracker.py Sat Dec 11 15:10:15 2010 @@ -3,7 +3,7 @@ Usage: call track(obj). """ -import autopath, sys, os +import autopath, sys, os, types import gc from pypy.translator.tool.graphpage import GraphPage, DotGen from pypy.tool.uid import uid @@ -39,7 +39,7 @@ if o2 is None: continue addedge(objectlist[i], o2) - id2typename[uid(o2)] = type(o2).__name__ + id2typename[uid(o2)] = self.shortrepr(o2) del o2 for o2 in self.get_referrers(objectlist[i]): if o2 is None: @@ -47,7 +47,7 @@ if type(o2) is list and o2 and o2[0] is MARKER: continue addedge(o2, objectlist[i]) - id2typename[uid(o2)] = type(o2).__name__ + id2typename[uid(o2)] = self.shortrepr(o2) del o2 for ids, label in edges.items(): @@ -82,13 +82,23 @@ return self.newpage(objectlist) def formatobject(self, o): + header = self.shortrepr(o, compact=False) + secondline = repr(o.__class__) + return header, secondline, repr(o) + + def shortrepr(self, o, compact=True): + t = type(o) + if t is types.FrameType: + if compact: + return 'frame %r' % (o.f_code.co_name,) + else: + return 'frame %r' % (o.f_code,) s = repr(o) if len(s) > 50: - linktext = s s = s[:20] + ' ... ' + s[-20:] - else: - linktext = '' - return type(o).__name__, s, linktext + if s.startswith('<') and s.endswith('>'): + s = s[1:-1] + return s def edgelabel(self, o1, o2): return '' From arigo at codespeak.net Sat Dec 11 15:45:43 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 15:45:43 +0100 (CET) Subject: [pypy-svn] r79978 - pypy/trunk/pypy/jit/metainterp Message-ID: <20101211144543.E192A282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 15:45:42 2010 New Revision: 79978 Modified: pypy/trunk/pypy/jit/metainterp/compile.py pypy/trunk/pypy/jit/metainterp/graphpage.py Log: Semi-hacks to still have around the information about which loop a jump goes to, even without having a strong reference. Modified: pypy/trunk/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/compile.py (original) +++ pypy/trunk/pypy/jit/metainterp/compile.py Sat Dec 11 15:45:42 2010 @@ -73,6 +73,8 @@ op.setdescr(None) # clear reference, mostly for tests # mostly for tests: make sure we don't keep a reference to the LoopToken loop.token = None + if not we_are_translated(): + loop._number = looptoken.number # ____________________________________________________________ Modified: pypy/trunk/pypy/jit/metainterp/graphpage.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/graphpage.py (original) +++ pypy/trunk/pypy/jit/metainterp/graphpage.py Sat Dec 11 15:45:42 2010 @@ -31,8 +31,10 @@ def compute(self, graphs, errmsg=None): resopgen = ResOpGen() for graph, highlight in graphs: - if hasattr(graph, 'token'): + if getattr(graph, 'token', None) is not None: resopgen.jumps_to_graphs[graph.token] = graph + if getattr(graph, '_number', None) is not None: + resopgen.jumps_to_graphs[graph._number] = graph for graph, highlight in graphs: resopgen.add_graph(graph, highlight) @@ -173,7 +175,10 @@ if tgt is None: tgt_g = graphindex else: - tgt = self.jumps_to_graphs.get(tgt) + if tgt in self.jumps_to_graphs: + tgt = self.jumps_to_graphs[tgt] + else: + tgt = self.jumps_to_graphs.get(tgt.number) if tgt is not None: tgt_g = self.graphs.index(tgt) if tgt_g != -1: From hakanardo at codespeak.net Sat Dec 11 15:48:11 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 11 Dec 2010 15:48:11 +0100 (CET) Subject: [pypy-svn] r79979 - pypy/branch/jit-unroll-loops/pypy/jit/metainterp Message-ID: <20101211144811.1E3F1282B9C@codespeak.net> Author: hakanardo Date: Sat Dec 11 15:48:09 2010 New Revision: 79979 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py Log: started to sort out merge failures Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py Sat Dec 11 15:48:09 2010 @@ -112,16 +112,17 @@ if full_preamble_needed or not loop.preamble.token.short_preamble: send_loop_to_backend(metainterp_sd, loop.preamble, "entry bridge") insert_loop_token(old_loop_tokens, loop.preamble.token) - record_loop_or_bridge(loop.preamble) jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( greenkey, loop.preamble.token) - return loop.preamble.token + token = loop.preamble.token + record_loop_or_bridge(loop.preamble) + return token else: send_loop_to_backend(metainterp_sd, loop, "loop") insert_loop_token(old_loop_tokens, loop_token) - record_loop_or_bridge(loop) jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( greenkey, loop.token) + record_loop_or_bridge(loop) return loop_token def insert_loop_token(old_loop_tokens, loop_token): From hakanardo at codespeak.net Sat Dec 11 15:50:20 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 11 Dec 2010 15:50:20 +0100 (CET) Subject: [pypy-svn] r79980 - in pypy/branch/jit-unroll-loops: . pypy pypy/jit/metainterp pypy/jit/metainterp/optimizeopt pypy/jit/metainterp/test pypy/module/array/benchmark pypy/module/array/test pypy/rlib pypy/rlib/test Message-ID: <20101211145020.5A078282B9C@codespeak.net> Author: hakanardo Date: Sat Dec 11 15:50:18 2010 New Revision: 79980 Modified: pypy/branch/jit-unroll-loops/ (props changed) pypy/branch/jit-unroll-loops/pypy/ (props changed) pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/graphpage.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/optimizeopt/optimizer.py (props changed) pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_send_nounroll.py (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/Makefile (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/intimg.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/intimgtst.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/intimgtst.py (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/loop.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/sum.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/sumtst.c (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/benchmark/sumtst.py (props changed) pypy/branch/jit-unroll-loops/pypy/module/array/test/test_array_old.py (props changed) pypy/branch/jit-unroll-loops/pypy/rlib/rerased.py (props changed) pypy/branch/jit-unroll-loops/pypy/rlib/test/test_rerased.py (props changed) Log: svn merge -r79977:HEAD svn+ssh://hakanardo at codespeak.net/svn/pypy/trunk Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py Sat Dec 11 15:50:18 2010 @@ -71,6 +71,8 @@ op.setdescr(None) # clear reference, mostly for tests # mostly for tests: make sure we don't keep a reference to the LoopToken loop.token = None + if not we_are_translated(): + loop._number = looptoken.number # ____________________________________________________________ Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/graphpage.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/graphpage.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/graphpage.py Sat Dec 11 15:50:18 2010 @@ -31,8 +31,10 @@ def compute(self, graphs, errmsg=None): resopgen = ResOpGen() for graph, highlight in graphs: - if hasattr(graph, 'token'): + if getattr(graph, 'token', None) is not None: resopgen.jumps_to_graphs[graph.token] = graph + if getattr(graph, '_number', None) is not None: + resopgen.jumps_to_graphs[graph._number] = graph for graph, highlight in graphs: resopgen.add_graph(graph, highlight) @@ -173,7 +175,10 @@ if tgt is None: tgt_g = graphindex else: - tgt = self.jumps_to_graphs.get(tgt) + if tgt in self.jumps_to_graphs: + tgt = self.jumps_to_graphs[tgt] + else: + tgt = self.jumps_to_graphs.get(tgt.number) if tgt is not None: tgt_g = self.graphs.index(tgt) if tgt_g != -1: From arigo at codespeak.net Sat Dec 11 16:34:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 16:34:08 +0100 (CET) Subject: [pypy-svn] r79981 - pypy/branch/jit-unroll-loops/pypy/jit/metainterp Message-ID: <20101211153408.A62D2282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 16:34:06 2010 New Revision: 79981 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/graphpage.py Log: More hacks of the same style as r79978. Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py Sat Dec 11 16:34:06 2010 @@ -69,10 +69,12 @@ if descr is not looptoken: looptoken.record_jump_to(descr) op.setdescr(None) # clear reference, mostly for tests + if not we_are_translated(): + op._jumptarget_number = descr.number # mostly for tests: make sure we don't keep a reference to the LoopToken loop.token = None if not we_are_translated(): - loop._number = looptoken.number + loop._looptoken_number = looptoken.number # ____________________________________________________________ Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/graphpage.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/graphpage.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/graphpage.py Sat Dec 11 16:34:06 2010 @@ -33,8 +33,8 @@ for graph, highlight in graphs: if getattr(graph, 'token', None) is not None: resopgen.jumps_to_graphs[graph.token] = graph - if getattr(graph, '_number', None) is not None: - resopgen.jumps_to_graphs[graph._number] = graph + if getattr(graph, '_looptoken_number', None) is not None: + resopgen.jumps_to_graphs[graph._looptoken_number] = graph for graph, highlight in graphs: resopgen.add_graph(graph, highlight) @@ -170,17 +170,21 @@ (graphindex, opindex)) break if op.getopnum() == rop.JUMP: - tgt = op.getdescr() tgt_g = -1 - if tgt is None: - tgt_g = graphindex + tgt = None + tgt_number = getattr(op, '_jumptarget_number', None) + if tgt_number is not None: + tgt = self.jumps_to_graphs.get(tgt_number) else: - if tgt in self.jumps_to_graphs: - tgt = self.jumps_to_graphs[tgt] + tgt_descr = op.getdescr() + if tgt_descr is None: + tgt_g = graphindex else: - tgt = self.jumps_to_graphs.get(tgt.number) - if tgt is not None: - tgt_g = self.graphs.index(tgt) + tgt = self.jumps_to_graphs.get(tgt_descr.number) + if tgt is None: + tgt = self.jumps_to_graphs.get(tgt_descr) + if tgt is not None: + tgt_g = self.graphs.index(tgt) if tgt_g != -1: self.genedge((graphindex, opstartindex), (tgt_g, 0), From hakanardo at codespeak.net Sat Dec 11 16:40:28 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 11 Dec 2010 16:40:28 +0100 (CET) Subject: [pypy-svn] r79982 - pypy/branch/jit-unroll-loops/pypy/jit/metainterp Message-ID: <20101211154028.54622282B9C@codespeak.net> Author: hakanardo Date: Sat Dec 11 16:40:26 2010 New Revision: 79982 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py Log: test_baisc passing Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py Sat Dec 11 16:40:26 2010 @@ -599,13 +599,16 @@ jmp = ResOperation(rop.JUMP, mini.inputargs[:], None, target) mini.operations = [jmp] descr = op.getdescr() - assert isinstance(descr, ResumeGuardDescr) + assert isinstance(descr, ResumeGuardDescr) + mini.token = make_loop_token(len(mini.inputargs), + metainterp.jitdriver_sd) #descr.compile_and_attach(metainterp, mini) if not we_are_translated(): descr._debug_suboperations = mini.operations send_bridge_to_backend(metainterp.staticdata, descr, - mini.inputargs, mini.operations) + mini.inputargs, mini.operations, + bridge.token) record_loop_or_bridge(mini) From hakanardo at codespeak.net Sat Dec 11 16:40:43 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 11 Dec 2010 16:40:43 +0100 (CET) Subject: [pypy-svn] r79983 - pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph Message-ID: <20101211154043.671CC282B9C@codespeak.net> Author: hakanardo Date: Sat Dec 11 16:40:41 2010 New Revision: 79983 Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/runner.py Log: test_baisc passing Modified: pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/backend/llgraph/runner.py Sat Dec 11 16:40:41 2010 @@ -202,13 +202,6 @@ llimpl.compile_add_fail_arg(c, var2index[box]) else: llimpl.compile_add_fail_arg(c, -1) - token = op.getjumptarget() - if token: - assert isinstance(token, history.LoopToken) - compiled_version = token._llgraph_compiled_version - llimpl.compile_add_guard_jump_target(c, compiled_version) - # Inform frontend that guard is patched to jump to token - op.setjumptarget(None) x = op.result if x is not None: From arigo at codespeak.net Sat Dec 11 16:55:37 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 16:55:37 +0100 (CET) Subject: [pypy-svn] r79984 - pypy/branch/more-posix Message-ID: <20101211155537.58DCC282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 16:55:35 2010 New Revision: 79984 Added: pypy/branch/more-posix/ - copied from r79983, pypy/trunk/ Log: A branch in which to add some more functions from the posix module. From arigo at codespeak.net Sat Dec 11 16:57:18 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 16:57:18 +0100 (CET) Subject: [pypy-svn] r79985 - in pypy/branch/more-posix/pypy/module/posix: . test Message-ID: <20101211155718.6D2D9282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 16:57:16 2010 New Revision: 79985 Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py pypy/branch/more-posix/pypy/module/posix/interp_posix.py pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py Log: Docstrings. mkfifo(). Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/__init__.py (original) +++ pypy/branch/more-posix/pypy/module/posix/__init__.py Sat Dec 11 16:57:16 2010 @@ -115,6 +115,8 @@ interpleveldefs['ttyname'] = 'interp_posix.ttyname' if hasattr(os, 'getloadavg'): interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' + if hasattr(os, 'mkfifo'): + interpleveldefs['mkfifo'] = 'interp_posix.mkfifo' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', Modified: pypy/branch/more-posix/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/more-posix/pypy/module/posix/interp_posix.py Sat Dec 11 16:57:16 2010 @@ -153,6 +153,7 @@ ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] def fsync(space, w_fd): + """Force write of file with filedescriptor to disk.""" fd = space.c_filedescriptor_w(w_fd) try: os.fsync(fd) @@ -161,6 +162,8 @@ fsync.unwrap_spec = [ObjSpace, W_Root] def fdatasync(space, w_fd): + """Force write of file with filedescriptor to disk. +Does not force update of metadata.""" fd = space.c_filedescriptor_w(w_fd) try: os.fdatasync(fd) @@ -169,6 +172,8 @@ fdatasync.unwrap_spec = [ObjSpace, W_Root] def fchdir(space, w_fd): + """Change to the directory of the given file descriptor. fildes must be +opened on a directory, not a file.""" fd = space.c_filedescriptor_w(w_fd) try: os.fchdir(fd) @@ -549,6 +554,14 @@ raise wrap_oserror(space, e) rename.unwrap_spec = [ObjSpace, W_Root, W_Root] +def mkfifo(space, w_filename, mode=0666): + """Create a FIFO (a POSIX named pipe).""" + try: + dispatch_filename(rposix.mkfifo)(space, w_filename, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_filename) +mkfifo.unwrap_spec = [ObjSpace, W_Root, "c_int"] + def umask(space, mask): "Set the current numeric umask and return the previous umask." prevmask = os.umask(mask) Modified: pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py Sat Dec 11 16:57:16 2010 @@ -659,6 +659,15 @@ f.close() os.chown(self.path, os.getuid(), os.getgid()) + if hasattr(os, 'mkfifo'): + def test_mkfifo(self): + os = self.posix + os.mkfifo(self.path2 + 'test_mkfifo', 0666) + st = os.lstat(self.path2 + 'test_mkfifo') + import stat + assert stat.S_ISFIFO(st.st_mode) + + class AppTestEnvironment(object): def setup_class(cls): cls.space = space From arigo at codespeak.net Sat Dec 11 16:58:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 16:58:16 +0100 (CET) Subject: [pypy-svn] r79986 - pypy/branch/more-posix/pypy/rlib Message-ID: <20101211155816.ED09D282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 16:58:15 2010 New Revision: 79986 Modified: pypy/branch/more-posix/pypy/rlib/rposix.py Log: mkfifo() needs to be added there too. Modified: pypy/branch/more-posix/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/more-posix/pypy/rlib/rposix.py (original) +++ pypy/branch/more-posix/pypy/rlib/rposix.py Sat Dec 11 16:58:15 2010 @@ -135,6 +135,13 @@ else: return os.rmdir(path.as_bytes()) + at specialize.argtype(0) +def mkfifo(path, mode): + if isinstance(path, str): + os.mkfifo(path, mode) + else: + os.mkfifo(path.as_bytes(), mode) + if os.name == 'nt': import nt def _getfullpathname(path): From hakanardo at codespeak.net Sat Dec 11 17:12:50 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Sat, 11 Dec 2010 17:12:50 +0100 (CET) Subject: [pypy-svn] r79987 - pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test Message-ID: <20101211161250.2C061282B9C@codespeak.net> Author: hakanardo Date: Sat Dec 11 17:12:46 2010 New Revision: 79987 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizebasic.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py Log: fixed tests Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizebasic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizebasic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizebasic.py Sat Dec 11 17:12:46 2010 @@ -39,7 +39,7 @@ b1 = BoxInt() opt = optimizeopt.Optimizer(FakeMetaInterpStaticData(LLtypeMixin.cpu), None) - fdescr = ResumeGuardDescr(None) + fdescr = ResumeGuardDescr() op = ResOperation(rop.GUARD_TRUE, ['dummy'], None, descr=fdescr) # setup rd data fi0 = resume.FrameInfo(None, "code0", 11) @@ -261,14 +261,13 @@ OptVirtualize, OptString, OptHeap, - OptFfiCall, Optimizer) optimizations = [OptIntBounds(), OptRewrite(), OptVirtualize(), OptString(), OptHeap(), - OptFfiCall(), + #OptFfiCall(), ] optimizer = Optimizer(metainterp_sd, loop, optimizations) optimizer.propagate_all_forward() @@ -796,6 +795,7 @@ """ expected = """ [i] + guard_no_exception() [] i1 = int_add(i, 3) i2 = call(i1, descr=nonwritedescr) guard_no_exception() [i1, i2] Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizeopt.py Sat Dec 11 17:12:46 2010 @@ -887,7 +887,7 @@ # note that 'guard_no_exception' at the very start must be kept # around: bridges may start with one. (In case of loops we could # remove it, but we probably don't care.) - expected = """ + preamble = """ [i] guard_no_exception() [] i1 = int_add(i, 3) @@ -896,7 +896,15 @@ i3 = call(i2, descr=nonwritedescr) jump(i1) """ - self.optimize_loop(ops, expected, expected) + expected = """ + [i] + i1 = int_add(i, 3) + i2 = call(i1, descr=nonwritedescr) + guard_no_exception() [i1, i2] + i3 = call(i2, descr=nonwritedescr) + jump(i1) + """ + self.optimize_loop(ops, expected, preamble) # ---------- @@ -2240,7 +2248,18 @@ setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) jump(p30) """ - self.optimize_loop(ops, 'Not', ops) + preamble = """ + [p9] + setfield_gc(ConstPtr(myptr), p9, descr=nextdescr) + jump() + """ + expected = """ + [] + p30 = new_with_vtable(ConstClass(node_vtable)) + setfield_gc(ConstPtr(myptr), p30, descr=nextdescr) + jump() + """ + self.optimize_loop(ops, expected, preamble) def test_invalid_loop_1(self): ops = """ From arigo at codespeak.net Sat Dec 11 17:13:46 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 17:13:46 +0100 (CET) Subject: [pypy-svn] r79988 - in pypy/branch/more-posix/pypy: module/posix module/posix/test rlib Message-ID: <20101211161346.3B038282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 17:13:44 2010 New Revision: 79988 Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py pypy/branch/more-posix/pypy/module/posix/interp_posix.py pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py pypy/branch/more-posix/pypy/rlib/rposix.py Log: os.mknod(). Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/__init__.py (original) +++ pypy/branch/more-posix/pypy/module/posix/__init__.py Sat Dec 11 17:13:44 2010 @@ -117,6 +117,8 @@ interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' if hasattr(os, 'mkfifo'): interpleveldefs['mkfifo'] = 'interp_posix.mkfifo' + if hasattr(os, 'mknod'): + interpleveldefs['mknod'] = 'interp_posix.mknod' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', Modified: pypy/branch/more-posix/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/more-posix/pypy/module/posix/interp_posix.py Sat Dec 11 17:13:44 2010 @@ -562,6 +562,19 @@ raise wrap_oserror2(space, e, w_filename) mkfifo.unwrap_spec = [ObjSpace, W_Root, "c_int"] +def mknod(space, w_filename, mode=0600, device=0): + """Create a filesystem node (file, device special file or named pipe) +named filename. mode specifies both the permissions to use and the +type of node to be created, being combined (bitwise OR) with one of +S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. For S_IFCHR and S_IFBLK, +device defines the newly created device special file (probably using +os.makedev()), otherwise it is ignored.""" + try: + dispatch_filename(rposix.mknod)(space, w_filename, mode, device) + except OSError, e: + raise wrap_oserror2(space, e, w_filename) +mknod.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] + def umask(space, mask): "Set the current numeric umask and return the previous umask." prevmask = os.umask(mask) Modified: pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py Sat Dec 11 17:13:44 2010 @@ -667,6 +667,35 @@ import stat assert stat.S_ISFIFO(st.st_mode) + if hasattr(os, 'mknod'): + def test_mknod(self): + import stat + os = self.posix + # not very useful: os.mknod() without specifying 'mode' + os.mknod(self.path2 + 'test_mknod-1') + st = os.lstat(self.path2 + 'test_mknod-1') + assert stat.S_ISREG(st.st_mode) + # os.mknod() with S_IFIFO + os.mknod(self.path2 + 'test_mknod-2', 0600 | stat.S_IFIFO) + st = os.lstat(self.path2 + 'test_mknod-2') + assert stat.S_ISFIFO(st.st_mode) + + def test_mknod_with_ifchr(self): + # os.mknod() with S_IFCHR + # -- usually requires root priviledges -- + os = self.posix + if hasattr(os.lstat('.'), 'st_rdev'): + import stat + try: + os.mknod(self.path2 + 'test_mknod-3', 0600 | stat.S_IFCHR, + 0x105) + except OSError, e: + skip("os.mknod() with S_IFCHR: got %r" % (e,)) + else: + st = os.lstat(self.path2 + 'test_mknod-3') + assert stat.S_ISCHR(st.st_mode) + assert st.st_rdev == 0x105 + class AppTestEnvironment(object): def setup_class(cls): Modified: pypy/branch/more-posix/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/more-posix/pypy/rlib/rposix.py (original) +++ pypy/branch/more-posix/pypy/rlib/rposix.py Sat Dec 11 17:13:44 2010 @@ -142,6 +142,13 @@ else: os.mkfifo(path.as_bytes(), mode) + at specialize.argtype(0) +def mknod(path, mode, device): + if isinstance(path, str): + os.mknod(path, mode, device) + else: + os.mknod(path.as_bytes(), mode, device) + if os.name == 'nt': import nt def _getfullpathname(path): From arigo at codespeak.net Sat Dec 11 17:33:32 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 17:33:32 +0100 (CET) Subject: [pypy-svn] r79989 - in pypy/branch/more-posix/pypy: rpython/module translator/c/test Message-ID: <20101211163332.709BB282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 17:33:30 2010 New Revision: 79989 Modified: pypy/branch/more-posix/pypy/rpython/module/ll_os.py pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py Log: Translate os.mknod() and os.mkfifo(). Modified: pypy/branch/more-posix/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/more-posix/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/more-posix/pypy/rpython/module/ll_os.py Sat Dec 11 17:33:30 2010 @@ -99,7 +99,7 @@ return 'll_os.ll_os_w' + name def registering_str_unicode(posixfunc, condition=True): - if not condition: + if not condition or posixfunc is None: return registering(None, condition=False) func_name = posixfunc.__name__ @@ -1323,6 +1323,33 @@ return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, export_name=traits.ll_os_name('rename')) + @registering_str_unicode(getattr(os, 'mkfifo', None)) + def register_os_mkfifo(self, traits): + os_mkfifo = self.llexternal(traits.posix_function_name('mkfifo'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) + + def mkfifo_llimpl(path, mode): + res = rffi.cast(lltype.Signed, os_mkfifo(path, mode)) + if res < 0: + raise OSError(rposix.get_errno(), "os_mkfifo failed") + + return extdef([traits.str, int], s_None, llimpl=mkfifo_llimpl, + export_name=traits.ll_os_name('mkfifo')) + + @registering_str_unicode(getattr(os, 'mknod', None)) + def register_os_mknod(self, traits): + os_mknod = self.llexternal(traits.posix_function_name('mknod'), + [traits.CCHARP, rffi.MODE_T, rffi.INT], + rffi.INT) # xxx: actually ^^^ dev_t + + def mknod_llimpl(path, mode, dev): + res = rffi.cast(lltype.Signed, os_mknod(path, mode, dev)) + if res < 0: + raise OSError(rposix.get_errno(), "os_mknod failed") + + return extdef([traits.str, int, int], s_None, llimpl=mknod_llimpl, + export_name=traits.ll_os_name('mknod')) + @registering(os.umask) def register_os_umask(self): os_umask = self.llexternal(underscore_on_windows+'umask', [rffi.MODE_T], rffi.MODE_T) Modified: pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py Sat Dec 11 17:33:30 2010 @@ -404,6 +404,28 @@ assert os.path.exists(tmpfile2) assert not os.path.exists(tmpfile1) +if hasattr(os, 'mkfifo'): + def test_os_mkfifo(): + tmpfile = str(udir.join('test_os_mkfifo.txt')) + def does_stuff(): + os.mkfifo(tmpfile, 0666) + f1 = compile(does_stuff, []) + f1() + import stat + st = os.lstat(tmpfile) + assert stat.S_ISFIFO(st.st_mode) + +if hasattr(os, 'mknod'): + def test_os_mknod(): + import stat + tmpfile = str(udir.join('test_os_mknod.txt')) + def does_stuff(): + os.mknod(tmpfile, 0600 | stat.S_IFIFO, 0) + f1 = compile(does_stuff, []) + f1() + st = os.lstat(tmpfile) + assert stat.S_ISFIFO(st.st_mode) + def test_os_umask(): def does_stuff(): mask1 = os.umask(0660) From arigo at codespeak.net Sat Dec 11 17:47:31 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 17:47:31 +0100 (CET) Subject: [pypy-svn] r79990 - in pypy/branch/more-posix/pypy/module/posix: . test Message-ID: <20101211164731.6F64B282BDC@codespeak.net> Author: arigo Date: Sat Dec 11 17:47:29 2010 New Revision: 79990 Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py pypy/branch/more-posix/pypy/module/posix/interp_posix.py pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py Log: os.nice(). Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/__init__.py (original) +++ pypy/branch/more-posix/pypy/module/posix/__init__.py Sat Dec 11 17:47:29 2010 @@ -119,6 +119,8 @@ interpleveldefs['mkfifo'] = 'interp_posix.mkfifo' if hasattr(os, 'mknod'): interpleveldefs['mknod'] = 'interp_posix.mknod' + if hasattr(os, 'nice'): + interpleveldefs['nice'] = 'interp_posix.nice' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', Modified: pypy/branch/more-posix/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/more-posix/pypy/module/posix/interp_posix.py Sat Dec 11 17:47:29 2010 @@ -1011,6 +1011,15 @@ space.wrap(load[2])]) getloadavg.unwrap_spec = [ObjSpace] +def nice(space, inc): + "Decrease the priority of process by inc and return the new priority." + try: + res = os.nice(inc) + except OSError, e: + raise wrap_oserror(space, e) + return space.wrap(res) +nice.unwrap_spec = [ObjSpace, "c_int"] + if _WIN: from pypy.rlib import rwin32 Modified: pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py Sat Dec 11 17:47:29 2010 @@ -696,6 +696,21 @@ assert stat.S_ISCHR(st.st_mode) assert st.st_rdev == 0x105 + if hasattr(os, 'nice') and hasattr(os, 'fork') and hasattr(os, 'waitpid'): + def test_nice(self): + os = self.posix + myprio = os.nice(0) + # + pid = os.fork() + if pid == 0: # in the child + res = os.nice(3) + os._exit(res) + # + pid1, status1 = os.waitpid(pid, 0) + assert pid1 == pid + assert os.WIFEXITED(status1) + assert os.WEXITSTATUS(status1) == myprio + 3 + class AppTestEnvironment(object): def setup_class(cls): From arigo at codespeak.net Sat Dec 11 17:55:45 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 17:55:45 +0100 (CET) Subject: [pypy-svn] r79991 - in pypy/branch/more-posix/pypy: rpython/module translator/c/test Message-ID: <20101211165545.00D75282BDC@codespeak.net> Author: arigo Date: Sat Dec 11 17:55:44 2010 New Revision: 79991 Modified: pypy/branch/more-posix/pypy/rpython/module/ll_os.py pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py Log: Translation of os.nice(). Modified: pypy/branch/more-posix/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/more-posix/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/more-posix/pypy/rpython/module/ll_os.py Sat Dec 11 17:55:44 2010 @@ -1471,6 +1471,25 @@ return extdef([int], s_None, llimpl=_exit_llimpl, export_name="ll_os.ll_os__exit") + @registering_if(os, 'nice') + def register_os_nice(self): + os_nice = self.llexternal('nice', [rffi.INT], rffi.INT) + + def nice_llimpl(inc): + # Assume that the system provides a standard-compliant version + # of nice() that returns the new priority. Nowadays, FreeBSD + # might be the last major non-compliant system (xxx check me). + rposix.set_errno(0) + res = rffi.cast(lltype.Signed, os_nice(inc)) + if res == -1: + err = rposix.get_errno() + if err != 0: + raise OSError(err, "os_nice failed") + return res + + return extdef([int], int, llimpl=nice_llimpl, + export_name="ll_os.ll_os_nice") + # --------------------------- os.stat & variants --------------------------- @registering(os.fstat) Modified: pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py Sat Dec 11 17:55:44 2010 @@ -4,6 +4,7 @@ from pypy.tool.udir import udir from pypy.rlib.rarithmetic import r_longlong from pypy.translator.c.test.test_genc import compile +from pypy.translator.c.test.test_standalone import StandaloneTests posix = __import__(os.name) # note: clock synchronizes itself! @@ -805,3 +806,19 @@ finally: os.chdir(localdir) assert res == True + +# ____________________________________________________________ + + +class TestExtFuncStandalone(StandaloneTests): + + if hasattr(os, 'nice'): + def test_os_nice(self): + def does_stuff(argv): + res = os.nice(3) + print 'os.nice returned', res + return 0 + t, cbuilder = self.compile(does_stuff) + data = cbuilder.cmdexec('') + res = os.nice(0) + 3 + assert data.startswith('os.nice returned %d\n' % res) From arigo at codespeak.net Sat Dec 11 17:59:04 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 17:59:04 +0100 (CET) Subject: [pypy-svn] r79992 - in pypy/branch/more-posix/pypy/module/posix: . test Message-ID: <20101211165904.CF95E282BDC@codespeak.net> Author: arigo Date: Sat Dec 11 17:59:03 2010 New Revision: 79992 Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py pypy/branch/more-posix/pypy/module/posix/interp_posix.py pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py Log: os.lchown(). Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/__init__.py (original) +++ pypy/branch/more-posix/pypy/module/posix/__init__.py Sat Dec 11 17:59:03 2010 @@ -71,6 +71,8 @@ if hasattr(os, 'chown'): interpleveldefs['chown'] = 'interp_posix.chown' + if hasattr(os, 'lchown'): + interpleveldefs['lchown'] = 'interp_posix.lchown' if hasattr(os, 'ftruncate'): interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' if hasattr(os, 'fsync'): Modified: pypy/branch/more-posix/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/more-posix/pypy/module/posix/interp_posix.py Sat Dec 11 17:59:03 2010 @@ -1000,6 +1000,14 @@ return space.w_None chown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] +def lchown(space, path, uid, gid): + try: + os.lchown(path, uid, gid) + except OSError, e: + raise wrap_oserror(space, e, path) + return space.w_None +lchown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] + def getloadavg(space): try: load = os.getloadavg() Modified: pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/more-posix/pypy/module/posix/test/test_posix2.py Sat Dec 11 17:59:03 2010 @@ -659,6 +659,14 @@ f.close() os.chown(self.path, os.getuid(), os.getgid()) + if hasattr(os, 'lchown'): + def test_lchown(self): + os = self.posix + os.unlink(self.path) + raises(OSError, os.lchown, self.path, os.getuid(), os.getgid()) + os.symlink('foobar', self.path) + os.lchown(self.path, os.getuid(), os.getgid()) + if hasattr(os, 'mkfifo'): def test_mkfifo(self): os = self.posix From arigo at codespeak.net Sat Dec 11 18:05:00 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 18:05:00 +0100 (CET) Subject: [pypy-svn] r79993 - in pypy/branch/more-posix/pypy: rpython/module translator/c/test Message-ID: <20101211170500.40536282BDC@codespeak.net> Author: arigo Date: Sat Dec 11 18:04:58 2010 New Revision: 79993 Modified: pypy/branch/more-posix/pypy/rpython/module/ll_os.py pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py Log: Translation of os.lchown(). Modified: pypy/branch/more-posix/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/more-posix/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/more-posix/pypy/rpython/module/ll_os.py Sat Dec 11 18:04:58 2010 @@ -1129,6 +1129,19 @@ return extdef([str, int, int], None, "ll_os.ll_os_chown", llimpl=os_chown_llimpl) + @registering_if(os, 'lchown') + def register_os_lchown(self): + os_lchown = self.llexternal('lchown',[rffi.CCHARP, rffi.INT, rffi.INT], + rffi.INT) + + def os_lchown_llimpl(path, uid, gid): + res = os_lchown(path, uid, gid) + if res == -1: + raise OSError(rposix.get_errno(), "os_lchown failed") + + return extdef([str, int, int], None, "ll_os.ll_os_lchown", + llimpl=os_lchown_llimpl) + @registering_if(os, 'readlink') def register_os_readlink(self): os_readlink = self.llexternal('readlink', Modified: pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py Sat Dec 11 18:04:58 2010 @@ -539,6 +539,28 @@ # for what reason do they want us to shift by 8? See the doc assert status1 >> 8 == 4 +if hasattr(os, 'chown') and hasattr(os, 'lchown'): + def test_os_chown_lchown(): + path1 = udir.join('test_os_chown_lchown-1.txt') + path2 = udir.join('test_os_chown_lchown-2.txt') + path1.write('foobar') + path2.mksymlinkto('some-broken-symlink') + tmpfile1 = str(path1) + tmpfile2 = str(path2) + def does_stuff(): + # xxx not really a test, just checks that they are callable + os.chown(tmpfile1, os.getuid(), os.getgid()) + os.lchown(tmpfile1, os.getuid(), os.getgid()) + os.lchown(tmpfile2, os.getuid(), os.getgid()) + try: + os.chown(tmpfile2, os.getuid(), os.getgid()) + except OSError: + pass + else: + raise AssertionError("os.chown(broken symlink) should raise") + f1 = compile(does_stuff, []) + f1() + # ____________________________________________________________ def _real_getenv(var): From arigo at codespeak.net Sat Dec 11 18:44:27 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 18:44:27 +0100 (CET) Subject: [pypy-svn] r79994 - in pypy/branch/more-posix/pypy: module/posix rpython/module translator/c/test Message-ID: <20101211174427.DB597282BDC@codespeak.net> Author: arigo Date: Sat Dec 11 18:44:26 2010 New Revision: 79994 Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py pypy/branch/more-posix/pypy/module/posix/interp_posix.py pypy/branch/more-posix/pypy/rpython/module/ll_os.py pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py Log: os.killpg() Modified: pypy/branch/more-posix/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/__init__.py (original) +++ pypy/branch/more-posix/pypy/module/posix/__init__.py Sat Dec 11 18:44:26 2010 @@ -88,6 +88,8 @@ if hasattr(os, 'kill') and sys.platform != 'win32': interpleveldefs['kill'] = 'interp_posix.kill' interpleveldefs['abort'] = 'interp_posix.abort' + if hasattr(os, 'killpg'): + interpleveldefs['killpg'] = 'interp_posix.killpg' if hasattr(os, 'getpid'): interpleveldefs['getpid'] = 'interp_posix.getpid' if hasattr(os, 'link'): Modified: pypy/branch/more-posix/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/more-posix/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/more-posix/pypy/module/posix/interp_posix.py Sat Dec 11 18:44:26 2010 @@ -598,6 +598,14 @@ raise wrap_oserror(space, e) kill.unwrap_spec = [ObjSpace, "c_int", "c_int"] +def killpg(space, pgid, sig): + "Kill a process group with a signal." + try: + os.killpg(pgid, sig) + except OSError, e: + raise wrap_oserror(space, e) +killpg.unwrap_spec = [ObjSpace, "c_int", "c_int"] + def abort(space): """Abort the interpreter immediately. This 'dumps core' or otherwise fails in the hardest way possible on the hosting operating system.""" Modified: pypy/branch/more-posix/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/more-posix/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/more-posix/pypy/rpython/module/ll_os.py Sat Dec 11 18:44:26 2010 @@ -1388,6 +1388,20 @@ return extdef([int, int], s_None, llimpl=kill_llimpl, export_name="ll_os.ll_os_kill") + @registering_if(os, 'killpg') + def register_os_killpg(self): + os_killpg = self.llexternal('killpg', [rffi.INT, rffi.INT], + rffi.INT) + + def killpg_llimpl(pid, sig): + res = rffi.cast(lltype.Signed, os_killpg(rffi.cast(rffi.INT, pid), + rffi.cast(rffi.INT, sig))) + if res < 0: + raise OSError(rposix.get_errno(), "os_killpg failed") + + return extdef([int, int], s_None, llimpl=killpg_llimpl, + export_name="ll_os.ll_os_killpg") + @registering_if(os, 'link') def register_os_link(self): os_link = self.llexternal('link', [rffi.CCHARP, rffi.CCHARP], Modified: pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/more-posix/pypy/translator/c/test/test_extfunc.py Sat Dec 11 18:44:26 2010 @@ -539,6 +539,40 @@ # for what reason do they want us to shift by 8? See the doc assert status1 >> 8 == 4 +if hasattr(os, 'kill'): + def test_kill_to_send_sigusr1(): + import signal + from pypy.module.signal import interp_signal + def does_stuff(): + interp_signal.pypysig_setflag(signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR1) + interp_signal.pypysig_ignore(signal.SIGUSR1) + while True: + n = interp_signal.pypysig_poll() + if n < 0 or n == signal.SIGUSR1: + break + return n + f1 = compile(does_stuff, []) + got_signal = f1() + assert got_signal == signal.SIGUSR1 + +if hasattr(os, 'killpg'): + def test_killpg(): + import signal + from pypy.module.signal import interp_signal + def does_stuff(): + interp_signal.pypysig_setflag(signal.SIGUSR1) + os.killpg(os.getpgrp(), signal.SIGUSR1) + interp_signal.pypysig_ignore(signal.SIGUSR1) + while True: + n = interp_signal.pypysig_poll() + if n < 0 or n == signal.SIGUSR1: + break + return n + f1 = compile(does_stuff, []) + got_signal = f1() + assert got_signal == signal.SIGUSR1 + if hasattr(os, 'chown') and hasattr(os, 'lchown'): def test_os_chown_lchown(): path1 = udir.join('test_os_chown_lchown-1.txt') From arigo at codespeak.net Sat Dec 11 18:53:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 18:53:17 +0100 (CET) Subject: [pypy-svn] r79995 - pypy/trunk/pypy/module/sys Message-ID: <20101211175317.F27F9282BDC@codespeak.net> Author: arigo Date: Sat Dec 11 18:53:16 2010 New Revision: 79995 Modified: pypy/trunk/pypy/module/sys/vm.py Log: Replace the deprecation message with something a bit clearer. The issue is that we can't arbitrarily add new 'deprecations' to the language. Modified: pypy/trunk/pypy/module/sys/vm.py ============================================================================== --- pypy/trunk/pypy/module/sys/vm.py (original) +++ pypy/trunk/pypy/module/sys/vm.py Sat Dec 11 18:53:16 2010 @@ -50,7 +50,7 @@ space.wrap("recursion limit must be positive")) # global recursion_limit # we need to do it without writing globals. - space.warn('setrecursionlimit deprecated', space.w_DeprecationWarning) + space.warn('setrecursionlimit() is ignored (and not needed) on PyPy', space.w_DeprecationWarning) space.sys.recursionlimit = new_limit def getrecursionlimit(space): From arigo at codespeak.net Sat Dec 11 19:15:52 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 19:15:52 +0100 (CET) Subject: [pypy-svn] r79996 - pypy/trunk/pypy/translator/goal Message-ID: <20101211181552.8EA17282BDC@codespeak.net> Author: arigo Date: Sat Dec 11 19:15:50 2010 New Revision: 79996 Added: pypy/trunk/pypy/translator/goal/targetsha1sum.py (contents, props changed) Log: The target I use to benchmark rlib/rsha.py. Added: pypy/trunk/pypy/translator/goal/targetsha1sum.py ============================================================================== --- (empty file) +++ pypy/trunk/pypy/translator/goal/targetsha1sum.py Sat Dec 11 19:15:50 2010 @@ -0,0 +1,29 @@ +#! /usr/bin/env python + +import os, sys +from pypy.rlib.rsha import RSHA + +# __________ Entry point __________ + +def entry_point(argv): + for filename in argv[1:]: + sha = RSHA() + fd = os.open(filename, os.O_RDONLY, 0) + while True: + buf = os.read(fd, 16384) + if not buf: break + sha.update(buf) + os.close(fd) + print sha.hexdigest(), filename + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +if __name__ == '__main__': + from sha import sha as RSHA + import sys + res = entry_point(sys.argv) + sys.exit(res) From arigo at codespeak.net Sat Dec 11 20:07:36 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sat, 11 Dec 2010 20:07:36 +0100 (CET) Subject: [pypy-svn] r79997 - pypy/trunk/pypy/rlib Message-ID: <20101211190736.BC994282B9C@codespeak.net> Author: arigo Date: Sat Dec 11 20:07:34 2010 New Revision: 79997 Modified: pypy/trunk/pypy/rlib/rmd5.py Log: A complete hack, but improve performance on 64-bit builds. It makes "sha" no longer 1.5x slower than on 32-bit, but about the same speed. I expect a similar speed-up for "md5". Modified: pypy/trunk/pypy/rlib/rmd5.py ============================================================================== --- pypy/trunk/pypy/rlib/rmd5.py (original) +++ pypy/trunk/pypy/rlib/rmd5.py Sat Dec 11 20:07:34 2010 @@ -29,20 +29,30 @@ if r_uint.BITS == 32: - def _mask(x): - "No-op on 32-bit platforms." - return x -else: - def _mask(x): - "Masks the r_uint value x to keep only the lowest 32 bits." - return x & r_uint(0xffffffff) - + def _rotateLeft(x, n): + "Rotate x (32 bit) left n bits circularly." + return (x << n) | (x >> (32-n)) -def _rotateLeft(x, n): - "Rotate x (32 bit) left n bits circularly." - - x = _mask(x) - return (x << n) | (x >> (32-n)) +else: + def _rotateLeft_emulator(x, n): + x &= 0xFFFFFFFF + return (x << n) | (x >> (32-n)) + + # ----- start of custom code, think about something better... ----- + from pypy.rpython.lltypesystem import lltype, rffi + from pypy.translator.tool.cbuild import ExternalCompilationInfo + eci = ExternalCompilationInfo(post_include_bits=[""" +static unsigned long pypy__rotateLeft(unsigned long x, long n) { + unsigned int x1 = x; /* arithmetic directly on int */ + int n1 = n; + return (x1 << n1) | (x1 >> (32-n1)); +} +"""]) + _rotateLeft = rffi.llexternal( + "pypy__rotateLeft", [lltype.Unsigned, lltype.Signed], lltype.Unsigned, + _callable=_rotateLeft_emulator, compilation_info=eci, + _nowrapper=True, pure_function=True) + # we expect the function _rotateLeft to be actually inlined def _state2string(a, b, c, d): From david at codespeak.net Sun Dec 12 11:29:12 2010 From: david at codespeak.net (david at codespeak.net) Date: Sun, 12 Dec 2010 11:29:12 +0100 (CET) Subject: [pypy-svn] r79998 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . helper Message-ID: <20101212102912.32586282B90@codespeak.net> Author: david Date: Sun Dec 12 11:29:08 2010 New Revision: 79998 Added: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/locations.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Log: Separete register allocation from codegeneration Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Sun Dec 12 11:29:08 2010 @@ -117,8 +117,6 @@ regs = rffi.cast(rffi.CCHARP, regs_loc) i = -1 fail_index = -1 - if self.debug: - import pdb; pdb.set_trace() while(True): i += 1 fail_index += 1 @@ -214,26 +212,26 @@ self.mc.ensure_can_fit(self.epilog_size) self.gen_func_epilog() - def _gen_path_to_exit_path(self, op, args, regalloc, fcond=c.AL): - + def _gen_path_to_exit_path(self, op, args, arglocs, fcond=c.AL): descr = op.getdescr() if op.getopnum() != rop.FINISH: assert isinstance(descr, AbstractFailDescr) - descr._arm_frame_depth = regalloc.frame_manager.frame_depth + descr._arm_frame_depth = arglocs[0].getint() reg = r.lr # XXX free this memory # XXX allocate correct amount of memory - mem = lltype.malloc(rffi.CArray(lltype.Char), len(args)*6+5, + mem = lltype.malloc(rffi.CArray(lltype.Char), (len(arglocs)-1)*6+5, flavor='raw', track_allocation=False) i = 0 j = 0 - while(i < len(args)): - if args[i]: - loc = regalloc.loc(args[i]) - if args[i].type == INT: + while i < len(args): + if arglocs[i+1]: + arg = args[i] + loc = arglocs[i+1] + if arg.type == INT: mem[j] = self.INT_TYPE j += 1 - elif args[i].type == REF: + elif arg.type == REF: mem[j] = self.REF_TYPE j += 1 else: @@ -243,7 +241,7 @@ mem[j] = chr(loc.value) j += 1 elif loc.is_imm(): - assert args[i].type == INT + assert arg.type == INT mem[j] = self.IMM_LOC self.encode32(mem, j+1, loc.getint()) j += 5 @@ -410,12 +408,15 @@ op = operations[i] opnum = op.getopnum() if self.can_merge_with_next_guard(op, i, operations): + arglocs = regalloc.operations_with_guard[opnum](regalloc, op, + operations[i+1], fcond) fcond = self.operations_with_guard[opnum](self, op, - operations[i+1], regalloc, fcond) + operations[i+1], arglocs, regalloc, fcond) i += 1 regalloc.position = i else: - fcond = self.operations[opnum](self, op, regalloc, fcond) + arglocs = regalloc.operations[opnum](regalloc, op, fcond) + fcond = self.operations[opnum](self, op, arglocs, regalloc, fcond) i += 1 def can_merge_with_next_guard(self, op, i, operations): @@ -456,40 +457,8 @@ def _dump_trace(self, name): self.mc._dump_trace(name) - def _check_imm_arg(self, arg, size=0xFF, allow_zero=True): - if isinstance(arg, ConstInt): - if allow_zero: - lower_bound = arg.getint() >= 0 - else: - lower_bound = arg.getint() > 0 - return arg.getint() <= size and lower_bound - return False - - def _ensure_value_is_boxed(self, thing, regalloc, forbidden_vars=[]): - # XXX create TempBox subclasses with the corresponding type flags and - # remove the overridden force_allocate_reg once done - box = None - loc = None - if isinstance(thing, Const): - if isinstance(thing, ConstInt): - box = TempInt() - elif isinstance(thing, ConstPtr): - box = TempPtr() - else: - box = TempBox() - loc = regalloc.force_allocate_reg(box, - forbidden_vars=forbidden_vars) - imm = regalloc.convert_to_imm(thing) - self.mc.gen_load_int(loc.value, imm.getint()) - else: - loc = regalloc.make_sure_var_in_reg(thing, - forbidden_vars=forbidden_vars, imm_fine=False) - box = thing - return loc, box - - - def _ensure_result_bit_extension(self, resloc, size, signed, regalloc): + def _ensure_result_bit_extension(self, resloc, size, signed): if size == 4: return if size == 1: @@ -500,11 +469,11 @@ self.mc.ASR_ri(resloc.value, resloc.value, 24) elif size == 2: if not signed: - t = TempBox() - loc = regalloc.force_allocate_reg(t) - self.mc.gen_load_int(loc.value, 0xFFFF) - self.mc.AND_rr(resloc.value, resloc.value, loc.value) - regalloc.possibly_free_var(t) + self.mc.LSL_ri(resloc.value, resloc.value, 16) + self.mc.LSR_ri(resloc.value, resloc.value, 16) + #self.mc.MOV_ri(r.ip.value, 0xFF) + #self.mc.ORR_ri(r.ip.value, 0xCFF) + #self.mc.AND_rr(resloc.value, resloc.value, r.ip.value) else: self.mc.LSL_ri(resloc.value, resloc.value, 16) self.mc.ASR_ri(resloc.value, resloc.value, 16) @@ -517,6 +486,11 @@ b.B(bridge_addr, some_reg=r.lr) # regalloc support + def load(self, loc, value): + assert loc.is_reg() + assert value.is_imm() + self.mc.gen_load_int(loc.value, value.getint()) + def regalloc_mov(self, prev_loc, loc): if prev_loc.is_imm(): # XXX check size of imm for current instr Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/assembler.py Sun Dec 12 11:29:08 2010 @@ -4,13 +4,9 @@ from pypy.jit.metainterp.history import ConstInt, BoxInt def gen_emit_op_unary_cmp(true_cond, false_cond): - def f(self, op, regalloc, fcond): + def f(self, op, arglocs, regalloc, fcond): assert fcond is not None - a0 = op.getarg(0) - reg, box = self._ensure_value_is_boxed(a0, regalloc) - res = regalloc.force_allocate_reg(op.result, [box]) - regalloc.possibly_free_vars([a0, box, op.result]) - + reg, res = arglocs self.mc.CMP_ri(reg.value, 0) self.mc.MOV_ri(res.value, 1, true_cond) self.mc.MOV_ri(res.value, 0, false_cond) @@ -20,30 +16,9 @@ def gen_emit_op_ri(opname, imm_size=0xFF, commutative=True, allow_zero=True): ri_op = getattr(AbstractARMv7Builder, '%s_ri' % opname) rr_op = getattr(AbstractARMv7Builder, '%s_rr' % opname) - def f(self, op, regalloc, fcond): + def f(self, op, arglocs, regalloc, fcond): assert fcond is not None - a0 = op.getarg(0) - a1 = op.getarg(1) - boxes = list(op.getarglist()) - imm_a0 = self._check_imm_arg(a0, imm_size, allow_zero=allow_zero) - imm_a1 = self._check_imm_arg(a1, imm_size, allow_zero=allow_zero) - if not imm_a0 and imm_a1: - l0, box = self._ensure_value_is_boxed(a0, regalloc) - boxes.append(box) - l1 = regalloc.make_sure_var_in_reg(a1, boxes) - elif commutative and imm_a0 and not imm_a1: - l1 = regalloc.make_sure_var_in_reg(a0, boxes) - l0, box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(box) - else: - l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) - boxes.append(box) - l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(box) - regalloc.possibly_free_vars(boxes) - res = regalloc.force_allocate_reg(op.result, boxes) - regalloc.possibly_free_var(op.result) - + l0, l1, res = arglocs if l1.is_imm(): ri_op(self.mc, res.value, l0.value, imm=l1.value, cond=fcond) else: @@ -53,47 +28,17 @@ def gen_emit_op_by_helper_call(opname): helper = getattr(AbstractARMv7Builder, opname) - def f(self, op, regalloc, fcond): + def f(self, op, arglocs, regalloc, fcond): assert fcond is not None - a0 = op.getarg(0) - a1 = op.getarg(1) - arg1 = regalloc.make_sure_var_in_reg(a0, selected_reg=r.r0) - arg2 = regalloc.make_sure_var_in_reg(a1, selected_reg=r.r1) - assert arg1 == r.r0 - assert arg2 == r.r1 - regalloc.before_call() + self.mc.PUSH([reg.value for reg in r.caller_resp][1:]) helper(self.mc, fcond) - regalloc.after_call(op.result) - - regalloc.possibly_free_var(a0) - regalloc.possibly_free_var(a1) - if op.result: - regalloc.possibly_free_var(op.result) + self.mc.POP([reg.value for reg in r.caller_resp][1:]) return fcond return f -def gen_emit_cmp_op(condition, inverse=False): - def f(self, op, regalloc, fcond): - assert fcond is not None - boxes = list(op.getarglist()) - if not inverse: - arg0, arg1 = boxes - else: - arg1, arg0 = boxes - # XXX consider swapping argumentes if arg0 is const - imm_a0 = self._check_imm_arg(arg0) - imm_a1 = self._check_imm_arg(arg1) - - l0, box = self._ensure_value_is_boxed(arg0, regalloc, forbidden_vars=boxes) - boxes.append(box) - if imm_a1 and not imm_a0: - l1 = regalloc.make_sure_var_in_reg(arg1, boxes) - else: - l1, box = self._ensure_value_is_boxed(arg1, regalloc, forbidden_vars=boxes) - boxes.append(box) - regalloc.possibly_free_vars(boxes) - res = regalloc.force_allocate_reg(op.result) - regalloc.possibly_free_var(op.result) +def gen_emit_cmp_op(condition): + def f(self, op, arglocs, regalloc, fcond): + l0, l1, res = arglocs inv = c.get_opposite_of(condition) if l1.is_imm(): Added: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py ============================================================================== --- (empty file) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py Sun Dec 12 11:29:08 2010 @@ -0,0 +1,84 @@ +from pypy.jit.backend.arm import conditions as c +from pypy.jit.backend.arm import registers as r +from pypy.jit.backend.arm.codebuilder import AbstractARMv7Builder +from pypy.jit.metainterp.history import ConstInt, BoxInt, Box + +def prepare_op_unary_cmp(): + def f(self, op, fcond): + assert fcond is not None + a0 = op.getarg(0) + reg, box = self._ensure_value_is_boxed(a0) + res = self.force_allocate_reg(op.result, [box]) + self.possibly_free_vars([a0, box, op.result]) + return [reg, res] + return f + +def prepare_op_ri(imm_size=0xFF, commutative=True, allow_zero=True): + def f(self, op, fcond): + assert fcond is not None + a0 = op.getarg(0) + a1 = op.getarg(1) + boxes = list(op.getarglist()) + imm_a0 = self._check_imm_arg(a0, imm_size, allow_zero=allow_zero) + imm_a1 = self._check_imm_arg(a1, imm_size, allow_zero=allow_zero) + if not imm_a0 and imm_a1: + l0, box = self._ensure_value_is_boxed(a0) + boxes.append(box) + l1 = self.make_sure_var_in_reg(a1, boxes) + elif commutative and imm_a0 and not imm_a1: + l1 = self.make_sure_var_in_reg(a0, boxes) + l0, box = self._ensure_value_is_boxed(a1, boxes) + boxes.append(box) + else: + l0, box = self._ensure_value_is_boxed(a0, boxes) + boxes.append(box) + l1, box = self._ensure_value_is_boxed(a1, boxes) + boxes.append(box) + self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result, boxes) + self.possibly_free_var(op.result) + return [l0, l1, res] + return f + +def prepare_op_by_helper_call(): + def f(self, op, fcond): + assert fcond is not None + a0 = op.getarg(0) + a1 = op.getarg(1) + arg1 = self.make_sure_var_in_reg(a0, selected_reg=r.r0) + arg2 = self.make_sure_var_in_reg(a1, selected_reg=r.r1) + assert arg1 == r.r0 + assert arg2 == r.r1 + if isinstance(a0, Box) and self.stays_alive(a0): + self.force_spill_var(a0) + self.after_call(op.result) + self.possibly_free_var(a0) + self.possibly_free_var(a1) + self.possibly_free_var(op.result) + return [] + return f + +def prepare_cmp_op(inverse=False): + def f(self, op, fcond): + assert fcond is not None + boxes = list(op.getarglist()) + if not inverse: + arg0, arg1 = boxes + else: + arg1, arg0 = boxes + # XXX consider swapping argumentes if arg0 is const + imm_a0 = self._check_imm_arg(arg0) + imm_a1 = self._check_imm_arg(arg1) + + l0, box = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes) + boxes.append(box) + if imm_a1 and not imm_a0: + l1 = self.make_sure_var_in_reg(arg1, boxes) + else: + l1, box = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes) + boxes.append(box) + self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + return [l0, l1, res] + return f Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/locations.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/locations.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/locations.py Sun Dec 12 11:29:08 2010 @@ -76,3 +76,6 @@ def as_key(self): return -self.position + +def imm(i): + return ImmLocation(i) Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Sun Dec 12 11:29:08 2010 @@ -15,7 +15,6 @@ from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr from pypy.jit.backend.llsupport.regalloc import compute_vars_longevity, TempBox -from pypy.jit.codewriter import heaptracker from pypy.jit.metainterp.history import (Const, ConstInt, BoxInt, Box, AbstractFailDescr, LoopToken, INT, FLOAT, REF) from pypy.jit.metainterp.resoperation import rop @@ -28,27 +27,8 @@ _mixin_ = True - def emit_op_int_add(self, op, regalloc, fcond): - #XXX check if neg values are supported for imm values - boxes = list(op.getarglist()) - a0, a1 = boxes - imm_a0 = self._check_imm_arg(a0) - imm_a1 = self._check_imm_arg(a1) - if not imm_a0 and imm_a1: - l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) - l1 = regalloc.make_sure_var_in_reg(a1, [a0]) - boxes.append(box) - elif imm_a0 and not imm_a1: - l0 = regalloc.make_sure_var_in_reg(a0) - l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(box) - else: - l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) - boxes.append(box) - l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(box) - res = regalloc.force_allocate_reg(op.result, boxes) - + def emit_op_int_add(self, op, arglocs, regalloc, fcond): + l0, l1, res = arglocs if l0.is_imm(): self.mc.ADD_ri(res.value, l1.value, imm=l0.value, s=1) elif l1.is_imm(): @@ -56,30 +36,10 @@ else: self.mc.ADD_rr(res.value, l0.value, l1.value, s=1) - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) return fcond - def emit_op_int_sub(self, op, regalloc, fcond): - #XXX check if neg values are supported for imm values - boxes = list(op.getarglist()) - a0, a1 = boxes - imm_a0 = self._check_imm_arg(a0) - imm_a1 = self._check_imm_arg(a1) - if not imm_a0 and imm_a1: - l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) - l1 = regalloc.make_sure_var_in_reg(a1, [a0]) - boxes.append(box) - elif imm_a0 and not imm_a1: - l0 = regalloc.make_sure_var_in_reg(a0) - l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(box) - else: - l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) - boxes.append(box) - l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(box) - res = regalloc.force_allocate_reg(op.result, boxes) + def emit_op_int_sub(self, op, arglocs, regalloc, fcond): + l0, l1, res = arglocs if l0.is_imm(): value = l0.getint() assert value >= 0 @@ -92,47 +52,28 @@ else: self.mc.SUB_rr(res.value, l0.value, l1.value, s=1) - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) return fcond - def emit_op_int_mul(self, op, regalloc, fcond): - boxes = list(op.getarglist()) - a0, a1 = boxes - - reg1, box = self._ensure_value_is_boxed(a0, regalloc, forbidden_vars=boxes) - boxes.append(box) - reg2, box = self._ensure_value_is_boxed(a1, regalloc, forbidden_vars=boxes) - boxes.append(box) - - res = regalloc.force_allocate_reg(op.result, boxes) + def emit_op_int_mul(self, op, arglocs, regalloc, fcond): + reg1, reg2, res = arglocs self.mc.MUL(res.value, reg1.value, reg2.value) - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) return fcond #ref: http://blogs.arm.com/software-enablement/detecting-overflow-from-mul/ - def emit_guard_int_mul_ovf(self, op, guard, regalloc, fcond): - boxes = list(op.getarglist()) - a0, a1 = boxes - - reg1, box = self._ensure_value_is_boxed(a0, regalloc, forbidden_vars=boxes) - boxes.append(box) - reg2, box = self._ensure_value_is_boxed(a1, regalloc, forbidden_vars=boxes) - boxes.append(box) - res = regalloc.force_allocate_reg(op.result, boxes) - + def emit_guard_int_mul_ovf(self, op, guard, arglocs, regalloc, fcond): + reg1 = arglocs[0] + reg2 = arglocs[1] + res = arglocs[2] + failargs = arglocs[3:] self.mc.SMULL(res.value, r.ip.value, reg1.value, reg2.value, cond=fcond) self.mc.CMP_rr(r.ip.value, res.value, shifttype=shift.ASR, imm=31, cond=fcond) if guard.getopnum() == rop.GUARD_OVERFLOW: - fcond = self._emit_guard(guard, regalloc, c.NE) + fcond = self._emit_guard(guard, failargs, c.NE) elif guard.getopnum() == rop.GUARD_NO_OVERFLOW: - fcond = self._emit_guard(guard, regalloc, c.EQ) + fcond = self._emit_guard(guard, failargs, c.EQ) else: assert 0 - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) return fcond emit_op_int_floordiv = gen_emit_op_by_helper_call('DIV') @@ -156,8 +97,8 @@ emit_op_uint_le = gen_emit_cmp_op(c.LS) emit_op_uint_gt = gen_emit_cmp_op(c.HI) - emit_op_uint_lt = gen_emit_cmp_op(c.HI, inverse=True) - emit_op_uint_ge = gen_emit_cmp_op(c.LS, inverse=True) + emit_op_uint_lt = gen_emit_cmp_op(c.HI) + emit_op_uint_ge = gen_emit_cmp_op(c.LS) emit_op_int_add_ovf = emit_op_int_add emit_op_int_sub_ovf = emit_op_int_sub @@ -166,7 +107,6 @@ emit_op_ptr_ne = emit_op_int_ne - class UnaryIntOpAssembler(object): _mixin_ = True @@ -174,20 +114,15 @@ emit_op_int_is_true = gen_emit_op_unary_cmp(c.NE, c.EQ) emit_op_int_is_zero = gen_emit_op_unary_cmp(c.EQ, c.NE) - def emit_op_int_invert(self, op, regalloc, fcond): - reg, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) - res = regalloc.force_allocate_reg(op.result, [box]) - regalloc.possibly_free_var(box) - regalloc.possibly_free_var(op.result) + def emit_op_int_invert(self, op, arglocs, regalloc, fcond): + reg, res = arglocs self.mc.MVN_rr(res.value, reg.value) return fcond #XXX check for a better way of doing this - def emit_op_int_neg(self, op, regalloc, fcond): - l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) - resloc = regalloc.force_allocate_reg(op.result, [box]) - regalloc.possibly_free_vars([box, op.result]) + def emit_op_int_neg(self, op, arglocs, regalloc, fcond): + l0, resloc = arglocs self.mc.MVN_ri(r.ip.value, imm=~-1) self.mc.MUL(resloc.value, l0.value, r.ip.value) @@ -198,7 +133,7 @@ _mixin_ = True guard_size = 10*WORD - def _emit_guard(self, op, regalloc, fcond, save_exc=False): + def _emit_guard(self, op, arglocs, fcond, save_exc=False): descr = op.getdescr() assert isinstance(descr, AbstractFailDescr) if not we_are_translated() and hasattr(op, 'getfailargs'): @@ -213,106 +148,76 @@ self.mc.BL(addr) self.mc.POP([reg.value for reg in r.caller_resp]) - memaddr = self._gen_path_to_exit_path(op, op.getfailargs(), regalloc) + memaddr = self._gen_path_to_exit_path(op, op.getfailargs(), arglocs) descr._failure_recovery_code = memaddr - regalloc.possibly_free_vars_for_op(op) return c.AL - def emit_op_guard_true(self, op, regalloc, fcond): - l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) + def emit_op_guard_true(self, op, arglocs, regalloc, fcond): + l0 = arglocs[0] + failargs = arglocs[1:] self.mc.CMP_ri(l0.value, 0) - fcond = self._emit_guard(op, regalloc, c.NE) - regalloc.possibly_free_var(box) + fcond = self._emit_guard(op, failargs, c.NE) return fcond - def emit_op_guard_false(self, op, regalloc, fcond): - l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) + def emit_op_guard_false(self, op, arglocs, regalloc, fcond): + l0 = arglocs[0] + failargs = arglocs[1:] self.mc.CMP_ri(l0.value, 0) - fcond = self._emit_guard(op, regalloc, c.EQ) - regalloc.possibly_free_var(box) + fcond = self._emit_guard(op, failargs, c.EQ) return fcond - def emit_op_guard_value(self, op, regalloc, fcond): - boxes = list(op.getarglist()) - a0, a1 = boxes - imm_a1 = self._check_imm_arg(a1) - l0, box = self._ensure_value_is_boxed(a0, regalloc, boxes) - boxes.append(box) - if not imm_a1: - l1, box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(box) - else: - l1 = regalloc.make_sure_var_in_reg(a1) + def emit_op_guard_value(self, op, arglocs, regalloc, fcond): + l0 = arglocs[0] + l1 = arglocs[1] + failargs = arglocs[2:] if l1.is_imm(): self.mc.CMP_ri(l0.value, l1.getint()) else: self.mc.CMP_rr(l0.value, l1.value) - fcond = self._emit_guard(op, regalloc, c.EQ) - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) + fcond = self._emit_guard(op, failargs, c.EQ) return fcond emit_op_guard_nonnull = emit_op_guard_true emit_op_guard_isnull = emit_op_guard_false - def emit_op_guard_no_overflow(self, op, regalloc, fcond): - return self._emit_guard(op, regalloc, c.VC) + def emit_op_guard_no_overflow(self, op, arglocs, regalloc, fcond): + return self._emit_guard(op, arglocs, c.VC) - def emit_op_guard_overflow(self, op, regalloc, fcond): - return self._emit_guard(op, regalloc, c.VS) + def emit_op_guard_overflow(self, op, arglocs, regalloc, fcond): + return self._emit_guard(op, arglocs, c.VS) # from ../x86/assembler.py:1265 - def emit_op_guard_class(self, op, regalloc, fcond): - locs = self._prepare_guard_class(op, regalloc, fcond) - self._cmp_guard_class(op, locs, regalloc, fcond) + def emit_op_guard_class(self, op, arglocs, regalloc, fcond): + self._cmp_guard_class(op, arglocs, regalloc, fcond) return fcond - def emit_op_guard_nonnull_class(self, op, regalloc, fcond): + def emit_op_guard_nonnull_class(self, op, arglocs, regalloc, fcond): offset = self.cpu.vtable_offset if offset is not None: self.mc.ensure_can_fit(self.guard_size+3*WORD) else: raise NotImplementedError - locs = self._prepare_guard_class(op, regalloc, fcond) - self.mc.CMP_ri(locs[0].value, 0) + self.mc.CMP_ri(arglocs[0].value, 0) if offset is not None: self.mc.ADD_ri(r.pc.value, r.pc.value, 2*WORD, cond=c.EQ) else: raise NotImplementedError - self._cmp_guard_class(op, locs, regalloc, fcond) + self._cmp_guard_class(op, arglocs, regalloc, fcond) return fcond - def _prepare_guard_class(self, op, regalloc, fcond): - assert isinstance(op.getarg(0), Box) - boxes = list(op.getarglist()) - - x, x_box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) - boxes.append(x_box) - - t = TempBox() - y = regalloc.force_allocate_reg(t, boxes) - boxes.append(t) - y_val = op.getarg(1).getint() - self.mc.gen_load_int(y.value, rffi.cast(lltype.Signed, y_val)) - - regalloc.possibly_free_vars(boxes) - return [x, y] - - def _cmp_guard_class(self, op, locs, regalloc, fcond): offset = self.cpu.vtable_offset - x, y = locs if offset is not None: assert offset == 0 - self.mc.LDR_ri(r.ip.value, x.value, offset) - self.mc.CMP_rr(r.ip.value, y.value) + self.mc.LDR_ri(r.ip.value, locs[0].value, offset) + self.mc.CMP_rr(r.ip.value, locs[1].value) else: raise NotImplementedError # XXX port from x86 backend once gc support is in place - return self._emit_guard(op, regalloc, c.EQ) + return self._emit_guard(op, locs[2:], c.EQ) @@ -320,28 +225,24 @@ _mixin_ = True - def emit_op_jump(self, op, regalloc, fcond): + def emit_op_jump(self, op, arglocs, regalloc, fcond): descr = op.getdescr() assert isinstance(descr, LoopToken) destlocs = descr._arm_arglocs - srclocs = [regalloc.loc(op.getarg(i)) for i in range(op.numargs())] - remap_frame_layout(self, srclocs, destlocs, r.ip) - loop_code = descr._arm_loop_code + + remap_frame_layout(self, arglocs, destlocs, r.ip) self.mc.B(loop_code, fcond) return fcond - def emit_op_finish(self, op, regalloc, fcond): - self._gen_path_to_exit_path(op, op.getarglist(), regalloc, c.AL) + def emit_op_finish(self, op, arglocs, regalloc, fcond): + self._gen_path_to_exit_path(op, op.getarglist(), arglocs, c.AL) return fcond - def emit_op_call(self, op, regalloc, fcond, spill_all_regs=False): - adr = rffi.cast(lltype.Signed, op.getarg(0).getint()) - args = op.getarglist()[1:] - cond = self._emit_call(adr, args, regalloc, fcond, op.result, spill_all_regs=spill_all_regs) - regalloc.possibly_free_vars(args) - if op.result: - regalloc.possibly_free_var(op.result) + def emit_op_call(self, op, args, regalloc, fcond, spill_all_regs=False): + adr = args[0] + cond = self._emit_call(adr, op.getarglist()[1:], regalloc, fcond, + op.result, spill_all_regs=spill_all_regs) descr = op.getdescr() #XXX Hack, Hack, Hack @@ -349,7 +250,7 @@ loc = regalloc.call_result_location(op.result) size = descr.get_result_size(False) signed = descr.is_result_signed() - self._ensure_result_bit_extension(loc, size, signed, regalloc) + self._ensure_result_bit_extension(loc, size, signed) return cond # XXX improve this interface @@ -381,7 +282,7 @@ n = stack_args*WORD self._adjust_sp(n, regalloc, fcond=fcond) for i in range(4, n_args): - reg, box = self._ensure_value_is_boxed(args[i], regalloc) + reg, box = regalloc._ensure_value_is_boxed(args[i], regalloc) self.mc.STR_ri(reg.value, r.sp.value, (i-4)*WORD) regalloc.possibly_free_var(box) @@ -405,56 +306,37 @@ self.mc.POP([reg.value for reg in r.caller_resp]) return fcond - def emit_op_same_as(self, op, regalloc, fcond): - resloc = regalloc.force_allocate_reg(op.result) - arg = op.getarg(0) - imm_arg = self._check_imm_arg(arg) - argloc = regalloc.make_sure_var_in_reg(arg, [op.result], imm_fine=imm_arg) + def emit_op_same_as(self, op, arglocs, regalloc, fcond): + argloc, resloc = arglocs if argloc.is_imm(): self.mc.MOV_ri(resloc.value, argloc.getint()) else: self.mc.MOV_rr(resloc.value, argloc.value) - regalloc.possibly_free_vars_for_op(op) - regalloc.possibly_free_var(op.result) return fcond def emit_op_cond_call_gc_wb(self, op, regalloc, fcond): #XXX implement once gc support is in place return fcond - def emit_op_guard_no_exception(self, op, regalloc, fcond): - loc, box = self._ensure_value_is_boxed( - ConstInt(self.cpu.pos_exception()), regalloc) - + def emit_op_guard_no_exception(self, op, arglocs, regalloc, fcond): + loc = arglocs[0] + failargs = arglocs[1:] self.mc.LDR_ri(loc.value, loc.value) self.mc.CMP_ri(loc.value, 0) - cond = self._emit_guard(op, regalloc, c.EQ, save_exc=True) - regalloc.possibly_free_var(box) + cond = self._emit_guard(op, failargs, c.EQ, save_exc=True) return cond - def emit_op_guard_exception(self, op, regalloc, fcond): - boxes = list(op.getarglist()) - arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) - loc, box = self._ensure_value_is_boxed(arg0, regalloc) - boxes.append(box) - loc1, box = self._ensure_value_is_boxed( - ConstInt(self.cpu.pos_exception()), regalloc, boxes) - boxes.append(box) - if op.result in regalloc.longevity: - resloc = regalloc.force_allocate_reg(op.result, boxes) - boxes.append(resloc) - else: - resloc = None - + def emit_op_guard_exception(self, op, arglocs, regalloc, fcond): + loc, loc1, resloc, pos_exc_value, pos_exception = arglocs[:5] + failargs = arglocs[5:] self.mc.LDR_ri(loc1.value, loc1.value) self.mc.CMP_rr(loc1.value, loc.value) - self._emit_guard(op, regalloc, c.EQ, save_exc=True) - regalloc.possibly_free_vars(boxes) - self.mc.gen_load_int(loc1.value, self.cpu.pos_exc_value(), fcond) + self._emit_guard(op, failargs, c.EQ, save_exc=True) + self.mc.gen_load_int(loc1.value, pos_exc_value.value, fcond) if resloc: self.mc.LDR_ri(resloc.value, loc1.value) - self.mc.gen_load_int(loc.value, self.cpu.pos_exception(), fcond) + self.mc.gen_load_int(loc.value, pos_exception.value, fcond) self.mc.MOV_ri(r.ip.value, 0) self.mc.STR_ri(r.ip.value, loc.value) self.mc.STR_ri(r.ip.value, loc1.value) @@ -468,52 +350,35 @@ _mixin_ = True - def emit_op_setfield_gc(self, op, regalloc, fcond): - boxes = list(op.getarglist()) - a0, a1 = boxes - ofs, size, ptr = self._unpack_fielddescr(op.getdescr()) - #ofs_loc = regalloc.make_sure_var_in_reg(ConstInt(ofs)) - #size_loc = regalloc.make_sure_var_in_reg(ofs) - base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc, boxes) - boxes.append(base_box) - value_loc, value_box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(value_box) - regalloc.possibly_free_vars(boxes) - if size == 4: - self.mc.STR_ri(value_loc.value, base_loc.value, ofs) - elif size == 2: - self.mc.STRH_ri(value_loc.value, base_loc.value, ofs) - elif size == 1: - self.mc.STRB_ri(value_loc.value, base_loc.value, ofs) + def emit_op_setfield_gc(self, op, arglocs, regalloc, fcond): + value_loc, base_loc, ofs, size = arglocs + if size.value == 4: + self.mc.STR_ri(value_loc.value, base_loc.value, ofs.value) + elif size.value == 2: + self.mc.STRH_ri(value_loc.value, base_loc.value, ofs.value) + elif size.value == 1: + self.mc.STRB_ri(value_loc.value, base_loc.value, ofs.value) else: assert 0 return fcond emit_op_setfield_raw = emit_op_setfield_gc - def emit_op_getfield_gc(self, op, regalloc, fcond): - a0 = op.getarg(0) - ofs, size, ptr = self._unpack_fielddescr(op.getdescr()) - # ofs_loc = regalloc.make_sure_var_in_reg(ConstInt(ofs)) - base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc) - regalloc.possibly_free_var(a0) - regalloc.possibly_free_var(base_box) - res = regalloc.force_allocate_reg(op.result, [a0]) - regalloc.possibly_free_var(op.result) - - if size == 4: - self.mc.LDR_ri(res.value, base_loc.value, ofs) - elif size == 2: - self.mc.LDRH_ri(res.value, base_loc.value, ofs) - elif size == 1: - self.mc.LDRB_ri(res.value, base_loc.value, ofs) + def emit_op_getfield_gc(self, op, arglocs, regalloc, fcond): + base_loc, ofs, res, size = arglocs + if size.value == 4: + self.mc.LDR_ri(res.value, base_loc.value, ofs.value) + elif size.value == 2: + self.mc.LDRH_ri(res.value, base_loc.value, ofs.value) + elif size.value == 1: + self.mc.LDRB_ri(res.value, base_loc.value, ofs.value) else: assert 0 #XXX Hack, Hack, Hack if not we_are_translated(): signed = op.getdescr().is_field_signed() - self._ensure_result_bit_extension(res, size, signed, regalloc) + self._ensure_result_bit_extension(res, size.value, signed) return fcond emit_op_getfield_raw = emit_op_getfield_gc @@ -522,56 +387,32 @@ - #XXX from ../x86/regalloc.py:791 - def _unpack_fielddescr(self, fielddescr): - assert isinstance(fielddescr, BaseFieldDescr) - ofs = fielddescr.offset - size = fielddescr.get_field_size(self.cpu.translate_support_code) - ptr = fielddescr.is_pointer_field() - return ofs, size, ptr class ArrayOpAssember(object): _mixin_ = True - def emit_op_arraylen_gc(self, op, regalloc, fcond): - arraydescr = op.getdescr() - assert isinstance(arraydescr, BaseArrayDescr) - ofs = arraydescr.get_ofs_length(self.cpu.translate_support_code) - arg = op.getarg(0) - base_loc, base_box = self._ensure_value_is_boxed(arg, regalloc) - res = regalloc.force_allocate_reg(op.result, forbidden_vars=[arg, base_box]) - regalloc.possibly_free_vars([arg, base_box, op.result]) - - self.mc.LDR_ri(res.value, base_loc.value, ofs) - return fcond - - def emit_op_setarrayitem_gc(self, op, regalloc, fcond): - a0, a1, a2 = boxes = list(op.getarglist()) - _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr()) - - base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc, boxes) - boxes.append(base_box) - ofs_loc, ofs_box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(ofs_box) - #XXX check if imm would be fine here - value_loc, value_box = self._ensure_value_is_boxed(a2, regalloc, boxes) - boxes.append(value_box) - regalloc.possibly_free_vars(boxes) + def emit_op_arraylen_gc(self, op, arglocs, regalloc, fcond): + res, base_loc, ofs = arglocs + self.mc.LDR_ri(res.value, base_loc.value, ofs.value) + return fcond + + def emit_op_setarrayitem_gc(self, op, arglocs, regalloc, fcond): + value_loc, base_loc, ofs_loc, scale, ofs = arglocs - if scale > 0: - self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale) + if scale.value > 0: + self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale.value) else: self.mc.MOV_rr(r.ip.value, ofs_loc.value) - if ofs > 0: - self.mc.ADD_ri(r.ip.value, r.ip.value, ofs) + if ofs.value > 0: + self.mc.ADD_ri(r.ip.value, r.ip.value, ofs.value) - if scale == 2: + if scale.value == 2: self.mc.STR_rr(value_loc.value, base_loc.value, r.ip.value, cond=fcond) - elif scale == 1: + elif scale.value == 1: self.mc.STRH_rr(value_loc.value, base_loc.value, r.ip.value, cond=fcond) - elif scale == 0: + elif scale.value == 0: self.mc.STRB_rr(value_loc.value, base_loc.value, r.ip.value, cond=fcond) else: assert 0 @@ -579,30 +420,20 @@ emit_op_setarrayitem_raw = emit_op_setarrayitem_gc - def emit_op_getarrayitem_gc(self, op, regalloc, fcond): - a0, a1 = boxes = list(op.getarglist()) - _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr()) - - base_loc, base_box = self._ensure_value_is_boxed(a0, regalloc, boxes) - boxes.append(base_box) - ofs_loc, ofs_box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(ofs_box) - res = regalloc.force_allocate_reg(op.result) - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) - - if scale > 0: - self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale) + def emit_op_getarrayitem_gc(self, op, arglocs, regalloc, fcond): + res, base_loc, ofs_loc, scale, ofs = arglocs + if scale.value > 0: + self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale.value) else: self.mc.MOV_rr(r.ip.value, ofs_loc.value) - if ofs > 0: - self.mc.ADD_ri(r.ip.value, r.ip.value, imm=ofs) + if ofs.value > 0: + self.mc.ADD_ri(r.ip.value, r.ip.value, imm=ofs.value) - if scale == 2: + if scale.value == 2: self.mc.LDR_rr(res.value, base_loc.value, r.ip.value, cond=fcond) - elif scale == 1: + elif scale.value == 1: self.mc.LDRH_rr(res.value, base_loc.value, r.ip.value, cond=fcond) - elif scale == 0: + elif scale.value == 0: self.mc.LDRB_rr(res.value, base_loc.value, r.ip.value, cond=fcond) else: assert 0 @@ -612,123 +443,62 @@ descr = op.getdescr() size = descr.get_item_size(False) signed = descr.is_item_signed() - self._ensure_result_bit_extension(res, size, signed, regalloc) + self._ensure_result_bit_extension(res, size, signed) return fcond emit_op_getarrayitem_raw = emit_op_getarrayitem_gc emit_op_getarrayitem_gc_pure = emit_op_getarrayitem_gc - #XXX from ../x86/regalloc.py:779 - def _unpack_arraydescr(self, arraydescr): - assert isinstance(arraydescr, BaseArrayDescr) - ofs_length = arraydescr.get_ofs_length(self.cpu.translate_support_code) - ofs = arraydescr.get_base_size(self.cpu.translate_support_code) - size = arraydescr.get_item_size(self.cpu.translate_support_code) - ptr = arraydescr.is_array_of_pointers() - scale = 0 - while (1 << scale) < size: - scale += 1 - assert (1 << scale) == size - return size, scale, ofs, ofs_length, ptr class StrOpAssembler(object): _mixin_ = True - def emit_op_strlen(self, op, regalloc, fcond): - l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) - boxes = [box] - - res = regalloc.force_allocate_reg(op.result, boxes) - boxes.append(op.result) - - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - ofs_box = ConstInt(ofs_length) - imm_ofs = self._check_imm_arg(ofs_box) - - if imm_ofs: - l1 = regalloc.make_sure_var_in_reg(ofs_box, boxes) - else: - l1, box1 = self._ensure_value_is_boxed(ofs_box, regalloc, boxes) - boxes.append(box1) - - regalloc.possibly_free_vars(boxes) - + def emit_op_strlen(self, op, arglocs, regalloc, fcond): + l0, l1, res = arglocs if l1.is_imm(): self.mc.LDR_ri(res.value, l0.value, l1.getint(), cond=fcond) else: self.mc.LDR_rr(res.value, l0.value, l1.value, cond=fcond) return fcond - def emit_op_strgetitem(self, op, regalloc, fcond): - boxes = list(op.getarglist()) - base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc) - boxes.append(box) - - a1 = boxes[1] - imm_a1 = self._check_imm_arg(a1) - if imm_a1: - ofs_loc = regalloc.make_sure_var_in_reg(a1, boxes) - else: - ofs_loc, box = self._ensure_value_is_boxed(a1, regalloc, boxes) - boxes.append(box) - res = regalloc.force_allocate_reg(op.result) - regalloc.possibly_free_vars(boxes) - - regalloc.possibly_free_var(op.result) - - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 + def emit_op_strgetitem(self, op, arglocs, regalloc, fcond): + res, base_loc, ofs_loc, basesize = arglocs if ofs_loc.is_imm(): self.mc.ADD_ri(r.ip.value, base_loc.value, ofs_loc.getint(), cond=fcond) else: self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value, cond=fcond) - self.mc.LDRB_ri(res.value, r.ip.value, basesize, cond=fcond) + self.mc.LDRB_ri(res.value, r.ip.value, basesize.value, cond=fcond) return fcond - def emit_op_strsetitem(self, op, regalloc, fcond): - boxes = list(op.getarglist()) - - base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) - boxes.append(box) - - ofs_loc, box = self._ensure_value_is_boxed(boxes[1], regalloc, boxes) - boxes.append(box) - - value_loc, box = self._ensure_value_is_boxed(boxes[2], regalloc, boxes) - boxes.append(box) - - regalloc.possibly_free_vars(boxes) - - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 + def emit_op_strsetitem(self, op, arglocs, regalloc, fcond): + value_loc, base_loc, ofs_loc, basesize = arglocs if ofs_loc.is_imm(): self.mc.ADD_ri(r.ip.value, base_loc.value, ofs_loc.getint(), cond=fcond) else: self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value, cond=fcond) - self.mc.STRB_ri(value_loc.value, r.ip.value, basesize, cond=fcond) + self.mc.STRB_ri(value_loc.value, r.ip.value, basesize.value, cond=fcond) return fcond #from ../x86/regalloc.py:928 ff. - def emit_op_copystrcontent(self, op, regalloc, fcond): + def emit_op_copystrcontent(self, op, arglocs, regalloc, fcond): + assert len(arglocs) == 0 self._emit_copystrcontent(op, regalloc, fcond, is_unicode=False) return fcond - def emit_op_copyunicodecontent(self, op, regalloc, fcond): + def emit_op_copyunicodecontent(self, op, arglocs, regalloc, fcond): + assert len(arglocs) == 0 self._emit_copystrcontent(op, regalloc, fcond, is_unicode=True) return fcond def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): # compute the source address args = list(op.getarglist()) - base_loc, box = self._ensure_value_is_boxed(args[0], regalloc, args) + base_loc, box = regalloc._ensure_value_is_boxed(args[0], args) args.append(box) - ofs_loc, box = self._ensure_value_is_boxed(args[2], regalloc, args) + ofs_loc, box = regalloc._ensure_value_is_boxed(args[2], args) args.append(box) assert args[0] is not args[1] # forbidden case of aliasing regalloc.possibly_free_var(args[0]) @@ -745,10 +515,10 @@ dstaddr_box = TempBox() dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, selected_reg=r.r0) forbidden_vars.append(dstaddr_box) - base_loc, box = self._ensure_value_is_boxed(args[1], regalloc, forbidden_vars) + base_loc, box = regalloc._ensure_value_is_boxed(args[1], forbidden_vars) args.append(box) forbidden_vars.append(box) - ofs_loc, box = self._ensure_value_is_boxed(args[3], regalloc, forbidden_vars) + ofs_loc, box = regalloc._ensure_value_is_boxed(args[3], forbidden_vars) args.append(box) assert base_loc.is_reg() assert ofs_loc.is_reg() @@ -760,7 +530,7 @@ # compute the length in bytes forbidden_vars = [srcaddr_box, dstaddr_box] - length_loc, length_box = self._ensure_value_is_boxed(args[4], regalloc, forbidden_vars) + length_loc, length_box = regalloc._ensure_value_is_boxed(args[4], forbidden_vars) args.append(length_box) if is_unicode: forbidden_vars = [srcaddr_box, dstaddr_box] @@ -822,84 +592,30 @@ _mixin_ = True - def emit_op_unicodelen(self, op, regalloc, fcond): - l0, box = self._ensure_value_is_boxed(op.getarg(0), regalloc) - boxes = [box] - res = regalloc.force_allocate_reg(op.result, boxes) - boxes.append(op.result) - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - ofs_box = ConstInt(ofs_length) - imm_ofs = self._check_imm_arg(ofs_box) - - if imm_ofs: - l1 = regalloc.make_sure_var_in_reg(ofs_box, boxes) - else: - l1, box1 = self._ensure_value_is_boxed(ofs_box, regalloc, boxes) - boxes.append(box1) - regalloc.possibly_free_vars(boxes) - - # XXX merge with strlen - if l1.is_imm(): - self.mc.LDR_ri(res.value, l0.value, l1.getint(), cond=fcond) - else: - self.mc.LDR_rr(res.value, l0.value, l1.value, cond=fcond) - return fcond - - def emit_op_unicodegetitem(self, op, regalloc, fcond): - boxes = list(op.getarglist()) - - base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) - boxes.append(box) - - ofs_loc, box = self._ensure_value_is_boxed(boxes[1], regalloc, boxes) - boxes.append(box) - - res = regalloc.force_allocate_reg(op.result) - - regalloc.possibly_free_vars(boxes) - regalloc.possibly_free_var(op.result) - - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - scale = itemsize/2 + emit_op_unicodelen = StrOpAssembler.emit_op_strlen + def emit_op_unicodegetitem(self, op, arglocs, regalloc, fcond): + res, base_loc, ofs_loc, scale, basesize, itemsize = arglocs self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value, cond=fcond, - imm=scale, shifttype=shift.LSL) - if scale == 2: - self.mc.LDR_ri(res.value, r.ip.value, basesize, cond=fcond) - elif scale == 1: - self.mc.LDRH_ri(res.value, r.ip.value, basesize, cond=fcond) + imm=scale.value, shifttype=shift.LSL) + if scale.value == 2: + self.mc.LDR_ri(res.value, r.ip.value, basesize.value, cond=fcond) + elif scale.value == 1: + self.mc.LDRH_ri(res.value, r.ip.value, basesize.value, cond=fcond) else: - assert 0, itemsize + assert 0, itemsize.value return fcond - def emit_op_unicodesetitem(self, op, regalloc, fcond): - boxes = list(op.getarglist()) - - base_loc, box = self._ensure_value_is_boxed(boxes[0], regalloc, boxes) - boxes.append(box) - - ofs_loc, box = self._ensure_value_is_boxed(boxes[1], regalloc, boxes) - boxes.append(box) - - value_loc, box = self._ensure_value_is_boxed(boxes[2], regalloc, boxes) - boxes.append(box) - - regalloc.possibly_free_vars(boxes) - - basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - scale = itemsize/2 - + def emit_op_unicodesetitem(self, op, arglocs, regalloc, fcond): + value_loc, base_loc, ofs_loc, scale, basesize, itemsize = arglocs self.mc.ADD_rr(r.ip.value, base_loc.value, ofs_loc.value, cond=fcond, - imm=scale, shifttype=shift.LSL) - if scale == 2: - self.mc.STR_ri(value_loc.value, r.ip.value, basesize, cond=fcond) - elif scale == 1: - self.mc.STRH_ri(value_loc.value, r.ip.value, basesize, cond=fcond) + imm=scale.value, shifttype=shift.LSL) + if scale.value == 2: + self.mc.STR_ri(value_loc.value, r.ip.value, basesize.value, cond=fcond) + elif scale.value == 1: + self.mc.STRH_ri(value_loc.value, r.ip.value, basesize.value, cond=fcond) else: - assert 0, itemsize + assert 0, itemsize.value return fcond @@ -907,18 +623,14 @@ _mixin_ = True - def emit_op_force_token(self, op, regalloc, fcond): - res_loc = regalloc.force_allocate_reg(op.result) + def emit_op_force_token(self, op, arglocs, regalloc, fcond): + res_loc = arglocs[0] self.mc.MOV_rr(res_loc.value, r.fp.value) return fcond # from: ../x86/assembler.py:1668 # XXX Split into some helper methods - def emit_guard_call_assembler(self, op, guard_op, regalloc, fcond): - faildescr = guard_op.getdescr() - fail_index = self.cpu.get_fail_descr_number(faildescr) - self._write_fail_index(fail_index, regalloc) - + def emit_guard_call_assembler(self, op, guard_op, arglocs, regalloc, fcond): descr = op.getdescr() assert isinstance(descr, LoopToken) @@ -1011,100 +723,57 @@ self.mc.LDR_ri(l0.value, r.fp.value) self.mc.CMP_ri(l0.value, 0) - self._emit_guard(guard_op, regalloc, c.GE) + self._emit_guard(guard_op, regalloc._prepare_guard(guard_op), c.GE) regalloc.possibly_free_var(t) regalloc.possibly_free_vars_for_op(op) if op.result: regalloc.possibly_free_var(op.result) return fcond - def emit_guard_call_may_force(self, op, guard_op, regalloc, fcond): - faildescr = guard_op.getdescr() - fail_index = self.cpu.get_fail_descr_number(faildescr) - self._write_fail_index(fail_index, regalloc) - - # force all reg values to be spilled when calling - fcond = self.emit_op_call(op, regalloc, fcond, spill_all_regs=True) + def emit_guard_call_may_force(self, op, guard_op, arglocs, regalloc, fcond): + self.mc.LDR_ri(r.ip.value, r.fp.value) + self.mc.CMP_ri(r.ip.value, 0) - t = TempBox() - l0 = regalloc.force_allocate_reg(t) - self.mc.LDR_ri(l0.value, r.fp.value) - self.mc.CMP_ri(l0.value, 0) - - self._emit_guard(guard_op, regalloc, c.GE) - regalloc.possibly_free_var(t) + self._emit_guard(guard_op, arglocs, c.GE) return fcond - def _write_fail_index(self, fail_index, regalloc): - t = TempBox() - l0 = regalloc.force_allocate_reg(t) - self.mc.gen_load_int(l0.value, fail_index) - self.mc.STR_ri(l0.value, r.fp.value) - regalloc.possibly_free_var(t) + def _write_fail_index(self, fail_index, temp_reg): + self.mc.gen_load_int(temp_reg.value, fail_index) + self.mc.STR_ri(temp_reg.value, r.fp.value) class AllocOpAssembler(object): _mixin_ = True - def _prepare_args_for_new_op(self, new_args, regalloc): - gc_ll_descr = self.cpu.gc_ll_descr - args = gc_ll_descr.args_for_new(new_args) - arglocs = [] - for arg in args: - t = TempBox() - l = regalloc.force_allocate_reg(t, arglocs) - self.mc.gen_load_int(l.value, arg) - arglocs.append(t) - return arglocs # from: ../x86/regalloc.py:750 + # called from regalloc # XXX kill this function at some point - def _malloc_varsize(self, ofs_items, ofs_length, itemsize, v, res_v, regalloc): - boxes = [v, res_v] - itemsize_box = ConstInt(itemsize) - ofs_items_box = ConstInt(ofs_items) - if self._check_imm_arg(ofs_items_box): - ofs_items_loc = regalloc.convert_to_imm(ofs_items_box) - else: - ofs_items_loc, ofs_items_box = self._ensure_value_is_boxed(ofs_items_box, regalloc, boxes) - boxes.append(ofs_items_box) - vloc, v = self._ensure_value_is_boxed(v, regalloc, [res_v]) - boxes.append(v) - size, size_box = self._ensure_value_is_boxed(itemsize_box, regalloc, boxes) - boxes.append(size_box) - + def _regalloc_malloc_varsize(self, size, size_box, vloc, ofs_items_loc, regalloc, result): self.mc.MUL(size.value, size.value, vloc.value) if ofs_items_loc.is_imm(): self.mc.ADD_ri(size.value, size.value, ofs_items_loc.value) else: self.mc.ADD_rr(size.value, size.value, ofs_items_loc.value) self._emit_call(self.malloc_func_addr, [size_box], regalloc, - result=res_v) - - base_loc = regalloc.make_sure_var_in_reg(res_v) - value_loc = regalloc.make_sure_var_in_reg(v) - regalloc.possibly_free_vars(boxes) - assert value_loc.is_reg() - assert base_loc.is_reg() - self.mc.STR_ri(value_loc.value, base_loc.value, ofs_length) - + result=result) - def emit_op_new(self, op, regalloc, fcond): - arglocs = self._prepare_args_for_new_op(op.getdescr(), regalloc) + def emit_op_new(self, op, arglocs, regalloc, fcond): self._emit_call(self.malloc_func_addr, arglocs, regalloc, result=op.result) + #XXX free args here, because _emit_call works on regalloc regalloc.possibly_free_vars(arglocs) regalloc.possibly_free_var(op.result) return fcond - def emit_op_new_with_vtable(self, op, regalloc, fcond): - classint = op.getarg(0).getint() - descrsize = heaptracker.vtable2descr(self.cpu, classint) - arglocs = self._prepare_args_for_new_op(descrsize, regalloc) - self._emit_call(self.malloc_func_addr, arglocs, + def emit_op_new_with_vtable(self, op, arglocs, regalloc, fcond): + classint = arglocs[-1].value + callargs = arglocs[:-1] + self._emit_call(self.malloc_func_addr, callargs, regalloc, result=op.result) self.set_vtable(op.result, classint, regalloc) - regalloc.possibly_free_vars(arglocs) + #XXX free args here, because _emit_call works on regalloc + regalloc.possibly_free_vars(callargs) regalloc.possibly_free_var(op.result) return fcond @@ -1120,57 +789,13 @@ self.mc.gen_load_int(loc_vtable.value, adr) self.mc.STR_ri(loc_vtable.value, loc.value, self.cpu.vtable_offset) - def emit_op_new_array(self, op, regalloc, fcond): - gc_ll_descr = self.cpu.gc_ll_descr - if gc_ll_descr.get_funcptr_for_newarray is not None: - raise NotImplementedError - #XXX make sure this path works - # framework GC - #args = self.cpu.gc_ll_descr.args_for_new_array(op.getdescr()) - #arglocs = [imm(x) for x in args] - #arglocs.append(self.loc(op.getarg(0))) - #return self._emit_call(self.malloc_array_func_addr, op.getarglist(), - # regalloc, result=op.result) - # boehm GC (XXX kill the following code at some point) - itemsize, scale, basesize, ofs_length, _ = ( - self._unpack_arraydescr(op.getdescr())) - self._malloc_varsize(basesize, ofs_length, itemsize, - op.getarg(0), op.result, regalloc) - return fcond - - - def emit_op_newstr(self, op, regalloc, fcond): - gc_ll_descr = self.cpu.gc_ll_descr - if gc_ll_descr.get_funcptr_for_newstr is not None: - raise NotImplementedError - # framework GC - #loc = self.loc(op.getarg(0)) - #return self._call(op, [loc]) - # boehm GC (XXX kill the following code at some point) - ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, - self.cpu.translate_support_code) - assert itemsize == 1 - self._malloc_varsize(ofs_items, ofs, itemsize, op.getarg(0), - op.result, regalloc) + def emit_op_new_array(self, op, arglocs, regalloc, fcond): + value_loc, base_loc, ofs_length = arglocs + self.mc.STR_ri(value_loc.value, base_loc.value, ofs_length.value) return fcond - - - def emit_op_newunicode(self, op, regalloc, fcond): - gc_ll_descr = self.cpu.gc_ll_descr - if gc_ll_descr.get_funcptr_for_newunicode is not None: - raise NotImplementedError - # framework GC - #loc = self.loc(op.getarg(0)) - #return self._call(op, [loc]) - # boehm GC (XXX kill the following code at some point) - ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, - self.cpu.translate_support_code) - self._malloc_varsize(ofs_items, ofs, itemsize, op.getarg(0), - op.result, regalloc) - return fcond + emit_op_newstr = emit_op_new_array + emit_op_newunicode = emit_op_new_array class ResOpAssembler(GuardOpAssembler, IntOpAsslember, OpAssembler, UnaryIntOpAssembler, Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Sun Dec 12 11:29:08 2010 @@ -2,8 +2,18 @@ RegisterManager, compute_vars_longevity, TempBox from pypy.jit.backend.arm import registers as r from pypy.jit.backend.arm import locations -from pypy.jit.metainterp.history import ConstInt, ConstPtr, REF, INT -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.jit.backend.arm.locations import imm +from pypy.jit.backend.arm.helper.regalloc import (prepare_op_by_helper_call, + prepare_op_unary_cmp, + prepare_op_ri, prepare_cmp_op) +from pypy.jit.metainterp.history import (Const, ConstInt, ConstPtr, Box, + BoxInt, BoxPtr, AbstractFailDescr, + INT, REF, FLOAT, LoopToken) +from pypy.jit.metainterp.resoperation import rop +from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr +from pypy.jit.backend.llsupport import symbolic +from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory +from pypy.jit.codewriter import heaptracker class TempInt(TempBox): type = INT @@ -15,6 +25,19 @@ def __repr__(self): return "" % (id(self),) +class ARMFrameManager(FrameManager): + def __init__(self): + FrameManager.__init__(self) + self.frame_depth = 1 + + @staticmethod + def frame_pos(loc, type): + # XXX for now we only have one word stack locs + return locations.StackLocation(loc) + +def void(self, op, fcond): + return [] + class ARMRegisterManager(RegisterManager): all_regs = r.all_regs box_types = None # or a list of acceptable types @@ -79,12 +102,635 @@ self._sync_var(var) del self.reg_bindings[var] -class ARMFrameManager(FrameManager): - def __init__(self): - FrameManager.__init__(self) - self.frame_depth = 1 + def _check_imm_arg(self, arg, size=0xFF, allow_zero=True): + if isinstance(arg, ConstInt): + if allow_zero: + lower_bound = arg.getint() >= 0 + else: + lower_bound = arg.getint() > 0 + return arg.getint() <= size and lower_bound + return False + + def _ensure_value_is_boxed(self, thing, forbidden_vars=[]): + # XXX create TempBox subclasses with the corresponding type flags and + # remove the overridden force_allocate_reg once done + box = None + loc = None + if isinstance(thing, Const): + if isinstance(thing, ConstInt): + box = TempInt() + elif isinstance(thing, ConstPtr): + box = TempPtr() + else: + box = TempBox() + loc = self.force_allocate_reg(box, + forbidden_vars=forbidden_vars) + imm = self.convert_to_imm(thing) + self.assembler.load(loc, imm) + else: + loc = self.make_sure_var_in_reg(thing, + forbidden_vars=forbidden_vars, imm_fine=False) + box = thing + return loc, box - @staticmethod - def frame_pos(loc, type): - # XXX for now we only have one word stack locs - return locations.StackLocation(loc) + + + + def prepare_op_int_add(self, op, fcond): + #XXX check if neg values are supported for imm values + boxes = list(op.getarglist()) + a0, a1 = boxes + imm_a0 = self._check_imm_arg(a0) + imm_a1 = self._check_imm_arg(a1) + if not imm_a0 and imm_a1: + l0, box = self._ensure_value_is_boxed(a0) + l1 = self.make_sure_var_in_reg(a1, [a0]) + boxes.append(box) + elif imm_a0 and not imm_a1: + l0 = self.make_sure_var_in_reg(a0) + l1, box = self._ensure_value_is_boxed(a1, [a0]) + boxes.append(box) + else: + l0, box = self._ensure_value_is_boxed(a0) + boxes.append(box) + l1, box = self._ensure_value_is_boxed(a1, [box]) + boxes.append(box) + res = self.force_allocate_reg(op.result, boxes) + self.possibly_free_vars(boxes) + self.possibly_free_var(op.result) + return [l0, l1, res] + + def prepare_op_int_sub(self, op, fcond): + #XXX check if neg values are supported for imm values + boxes = list(op.getarglist()) + a0, a1 = boxes + imm_a0 = self._check_imm_arg(a0) + imm_a1 = self._check_imm_arg(a1) + if not imm_a0 and imm_a1: + l0, box = self._ensure_value_is_boxed(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, [a0]) + boxes.append(box) + elif imm_a0 and not imm_a1: + l0 = self.make_sure_var_in_reg(a0) + l1, box = self._ensure_value_is_boxed(a1, boxes) + boxes.append(box) + else: + l0, box = self._ensure_value_is_boxed(a0, boxes) + boxes.append(box) + l1, box = self._ensure_value_is_boxed(a1, boxes) + boxes.append(box) + res = self.force_allocate_reg(op.result, boxes) + self.possibly_free_vars(boxes) + self.possibly_free_var(op.result) + return [l0, l1, res] + + def prepare_op_int_mul(self, op, fcond): + boxes = list(op.getarglist()) + a0, a1 = boxes + + reg1, box = self._ensure_value_is_boxed(a0, forbidden_vars=boxes) + boxes.append(box) + reg2, box = self._ensure_value_is_boxed(a1, forbidden_vars=boxes) + boxes.append(box) + + res = self.force_allocate_reg(op.result, boxes) + self.possibly_free_vars(boxes) + self.possibly_free_var(op.result) + return [reg1, reg2, res] + + def prepare_guard_int_mul_ovf(self, op, guard, fcond): + args = [] + boxes = list(op.getarglist()) + a0, a1 = boxes + + reg1, box = self._ensure_value_is_boxed(a0,forbidden_vars=boxes) + boxes.append(box) + reg2, box = self._ensure_value_is_boxed(a1,forbidden_vars=boxes) + boxes.append(box) + self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result, boxes) + + args.append(reg1) + args.append(reg2) + args.append(res) + args = self._prepare_guard(guard, args) + self.possibly_free_var(op.result) + return args + + + prepare_op_int_floordiv = prepare_op_by_helper_call() + prepare_op_int_mod = prepare_op_by_helper_call() + prepare_op_uint_floordiv = prepare_op_by_helper_call() + + prepare_op_int_and = prepare_op_ri() + prepare_op_int_or = prepare_op_ri() + prepare_op_int_xor = prepare_op_ri() + prepare_op_int_lshift = prepare_op_ri(imm_size=0x1F, allow_zero=False, commutative=False) + prepare_op_int_rshift = prepare_op_ri(imm_size=0x1F, allow_zero=False, commutative=False) + prepare_op_uint_rshift = prepare_op_ri(imm_size=0x1F, allow_zero=False, commutative=False) + + prepare_op_int_lt = prepare_cmp_op() + prepare_op_int_le = prepare_cmp_op() + prepare_op_int_eq = prepare_cmp_op() + prepare_op_int_ne = prepare_cmp_op() + prepare_op_int_gt = prepare_cmp_op() + prepare_op_int_ge = prepare_cmp_op() + + prepare_op_uint_le = prepare_cmp_op() + prepare_op_uint_gt = prepare_cmp_op() + + prepare_op_uint_lt = prepare_cmp_op(inverse=True) + prepare_op_uint_ge = prepare_cmp_op(inverse=True) + + prepare_op_int_add_ovf = prepare_op_int_add + prepare_op_int_sub_ovf = prepare_op_int_sub + + prepare_op_ptr_eq = prepare_op_int_eq + prepare_op_ptr_ne = prepare_op_int_ne + + prepare_op_int_is_true = prepare_op_unary_cmp() + prepare_op_int_is_zero = prepare_op_unary_cmp() + + def prepare_op_int_neg(self, op, fcond): + l0, box = self._ensure_value_is_boxed(op.getarg(0)) + resloc = self.force_allocate_reg(op.result, [box]) + self.possibly_free_vars([box, op.result]) + return [l0, resloc] + + prepare_op_int_invert = prepare_op_int_neg + + def prepare_op_call(self, op, fcond): + args = [rffi.cast(lltype.Signed, op.getarg(0).getint())] + return args + + def _prepare_guard(self, op, args=None): + if args is None: + args = [] + args.append(imm(self.frame_manager.frame_depth)) + for arg in op.getfailargs(): + if arg: + args.append(self.loc(arg)) + else: + args.append(None) + return args + + def prepare_op_finish(self, op, fcond): + args = [imm(self.frame_manager.frame_depth)] + for i in range(op.numargs()): + arg = op.getarg(i) + if arg: + args.append(self.loc(arg)) + else: + args.append(None) + return args + + def prepare_op_guard_true(self, op, fcond): + l0, box = self._ensure_value_is_boxed(op.getarg(0)) + args = self._prepare_guard(op, [l0]) + self.possibly_free_var(box) + return args + + prepare_op_guard_false = prepare_op_guard_true + prepare_op_guard_nonnull = prepare_op_guard_true + prepare_op_guard_isnull = prepare_op_guard_true + + def prepare_op_guard_value(self, op, fcond): + boxes = list(op.getarglist()) + a0, a1 = boxes + imm_a1 = self._check_imm_arg(a1) + l0, box = self._ensure_value_is_boxed(a0, boxes) + boxes.append(box) + if not imm_a1: + l1, box = self._ensure_value_is_boxed(a1,boxes) + boxes.append(box) + else: + l1 = self.make_sure_var_in_reg(a1) + self.possibly_free_vars(boxes) + assert op.result is None + return self._prepare_guard(op, [l0, l1]) + + def prepare_op_guard_no_overflow(self, op, fcond): + return self._prepare_guard(op) + + prepare_op_guard_overflow = prepare_op_guard_no_overflow + + def prepare_op_guard_exception(self, op, fcond): + boxes = list(op.getarglist()) + arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) + loc, box = self._ensure_value_is_boxed(arg0) + boxes.append(box) + loc1, box = self._ensure_value_is_boxed( + ConstInt(self.assembler.cpu.pos_exception()), boxes) + boxes.append(box) + if op.result in self.longevity: + resloc = self.force_allocate_reg(op.result, boxes) + boxes.append(resloc) + else: + resloc = None + self.possibly_free_vars(boxes) + pos_exc_value = imm(self.assembler.cpu.pos_exc_value()) + pos_exception = imm(self.assembler.cpu.pos_exception()) + return self._prepare_guard(op, [loc, loc1, resloc, pos_exc_value, pos_exception]) + + def prepare_op_guard_no_exception(self, op, fcond): + loc, box = self._ensure_value_is_boxed( + ConstInt(self.assembler.cpu.pos_exception())) + self.possibly_free_var(box) + return self._prepare_guard(op, [loc]) + + def prepare_op_guard_class(self, op, fcond): + return self._prepare_guard_class(op, fcond) + + prepare_op_guard_nonnull_class = prepare_op_guard_class + + def _prepare_guard_class(self, op, fcond): + assert isinstance(op.getarg(0), Box) + boxes = list(op.getarglist()) + + x, x_box = self._ensure_value_is_boxed(boxes[0], boxes) + boxes.append(x_box) + + t = TempBox() + y = self.force_allocate_reg(t, boxes) + boxes.append(t) + y_val = rffi.cast(lltype.Signed, op.getarg(1).getint()) + self.assembler.load(y, imm(y_val)) + + self.possibly_free_vars(boxes) + return self._prepare_guard(op, [x, y]) + + def prepare_op_jump(self, op, fcond): + descr = op.getdescr() + assert isinstance(descr, LoopToken) + return [self.loc(op.getarg(i)) for i in range(op.numargs())] + + + def prepare_op_setfield_gc(self, op, fcond): + boxes = list(op.getarglist()) + a0, a1 = boxes + ofs, size, ptr = self._unpack_fielddescr(op.getdescr()) + base_loc, base_box = self._ensure_value_is_boxed(a0, boxes) + boxes.append(base_box) + value_loc, value_box = self._ensure_value_is_boxed(a1, boxes) + boxes.append(value_box) + self.possibly_free_vars(boxes) + return [value_loc, base_loc, imm(ofs), imm(size)] + + prepare_op_setfield_raw = prepare_op_setfield_gc + + def prepare_op_getfield_gc(self, op, fcond): + a0 = op.getarg(0) + ofs, size, ptr = self._unpack_fielddescr(op.getdescr()) + base_loc, base_box = self._ensure_value_is_boxed(a0) + self.possibly_free_var(a0) + self.possibly_free_var(base_box) + res = self.force_allocate_reg(op.result, [a0]) + self.possibly_free_var(op.result) + return [base_loc, imm(ofs), res, imm(size)] + + prepare_op_getfield_raw = prepare_op_getfield_gc + prepare_op_getfield_raw_pure = prepare_op_getfield_gc + prepare_op_getfield_gc_pure = prepare_op_getfield_gc + + def prepare_op_arraylen_gc(self, op, fcond): + arraydescr = op.getdescr() + assert isinstance(arraydescr, BaseArrayDescr) + ofs = arraydescr.get_ofs_length(self.assembler.cpu.translate_support_code) + arg = op.getarg(0) + base_loc, base_box = self._ensure_value_is_boxed(arg) + self.possibly_free_vars([arg, base_box]) + + res = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + return [res, base_loc, imm(ofs)] + + def prepare_op_setarrayitem_gc(self, op, fcond): + a0, a1, a2 = boxes = list(op.getarglist()) + _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr()) + + base_loc, base_box = self._ensure_value_is_boxed(a0, boxes) + boxes.append(base_box) + ofs_loc, ofs_box = self._ensure_value_is_boxed(a1, boxes) + boxes.append(ofs_box) + #XXX check if imm would be fine here + value_loc, value_box = self._ensure_value_is_boxed(a2, boxes) + boxes.append(value_box) + self.possibly_free_vars(boxes) + return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)] + prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc + + def prepare_op_getarrayitem_gc(self, op, fcond): + a0, a1 = boxes = list(op.getarglist()) + _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr()) + + base_loc, base_box = self._ensure_value_is_boxed(a0, boxes) + boxes.append(base_box) + ofs_loc, ofs_box = self._ensure_value_is_boxed(a1, boxes) + boxes.append(ofs_box) + self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + return [res, base_loc, ofs_loc, imm(scale), imm(ofs)] + + prepare_op_getarrayitem_raw = prepare_op_getarrayitem_gc + prepare_op_getarrayitem_gc_pure = prepare_op_getarrayitem_gc + + def prepare_op_strlen(self, op, fcond): + l0, box = self._ensure_value_is_boxed(op.getarg(0)) + boxes = [box] + + + basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, + self.assembler.cpu.translate_support_code) + ofs_box = ConstInt(ofs_length) + imm_ofs = self._check_imm_arg(ofs_box) + + if imm_ofs: + l1 = self.make_sure_var_in_reg(ofs_box, boxes) + else: + l1, box1 = self._ensure_value_is_boxed(ofs_box, boxes) + boxes.append(box1) + + self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + return [l0, l1, res] + + def prepare_op_strgetitem(self, op, fcond): + boxes = list(op.getarglist()) + base_loc, box = self._ensure_value_is_boxed(boxes[0]) + boxes.append(box) + + a1 = boxes[1] + imm_a1 = self._check_imm_arg(a1) + if imm_a1: + ofs_loc = self.make_sure_var_in_reg(a1, boxes) + else: + ofs_loc, box = self._ensure_value_is_boxed(a1, boxes) + boxes.append(box) + + self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + + basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, + self.assembler.cpu.translate_support_code) + assert itemsize == 1 + return [res, base_loc, ofs_loc, imm(basesize)] + + def prepare_op_strsetitem(self, op, fcond): + boxes = list(op.getarglist()) + + base_loc, box = self._ensure_value_is_boxed(boxes[0], boxes) + boxes.append(box) + + ofs_loc, box = self._ensure_value_is_boxed(boxes[1], boxes) + boxes.append(box) + + value_loc, box = self._ensure_value_is_boxed(boxes[2], boxes) + boxes.append(box) + + self.possibly_free_vars(boxes) + + basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, + self.assembler.cpu.translate_support_code) + assert itemsize == 1 + return [value_loc, base_loc, ofs_loc, imm(basesize)] + + prepare_op_copystrcontent = void + prepare_op_copyunicodecontent = void + + def prepare_op_unicodelen(self, op, fcond): + l0, box = self._ensure_value_is_boxed(op.getarg(0)) + boxes = [box] + basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, + self.assembler.cpu.translate_support_code) + ofs_box = ConstInt(ofs_length) + imm_ofs = self._check_imm_arg(ofs_box) + + if imm_ofs: + l1 = imm(ofs_length) + else: + l1, box1 = self._ensure_value_is_boxed(ofs_box, boxes) + boxes.append(box1) + + self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + return [l0, l1, res] + + def prepare_op_unicodegetitem(self, op, fcond): + boxes = list(op.getarglist()) + base_loc, box = self._ensure_value_is_boxed(boxes[0], boxes) + boxes.append(box) + ofs_loc, box = self._ensure_value_is_boxed(boxes[1], boxes) + boxes.append(box) + self.possibly_free_vars(boxes) + + res = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + + basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, + self.assembler.cpu.translate_support_code) + scale = itemsize/2 + return [res, base_loc, ofs_loc, imm(scale), imm(basesize), imm(itemsize)] + + def prepare_op_unicodesetitem(self, op, fcond): + boxes = list(op.getarglist()) + base_loc, box = self._ensure_value_is_boxed(boxes[0], boxes) + boxes.append(box) + ofs_loc, box = self._ensure_value_is_boxed(boxes[1], boxes) + boxes.append(box) + value_loc, box = self._ensure_value_is_boxed(boxes[2], boxes) + boxes.append(box) + + self.possibly_free_vars(boxes) + + basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, + self.assembler.cpu.translate_support_code) + scale = itemsize/2 + return [value_loc, base_loc, ofs_loc, imm(scale), imm(basesize), imm(itemsize)] + + def prepare_op_same_as(self, op, fcond): + arg = op.getarg(0) + imm_arg = self._check_imm_arg(arg) + if imm_arg: + argloc = self.make_sure_var_in_reg(arg) + else: + argloc, box = self._ensure_value_is_boxed(arg) + self.possibly_free_var(box) + self.possibly_free_vars_for_op(op) + + resloc = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + return [argloc, resloc] + + def prepare_op_new(self, op, fcond): + arglocs = self._prepare_args_for_new_op(op.getdescr()) + #XXX args are freed in assembler._emit_call + #self.possibly_free_vars(arglocs) + self.possibly_free_var(op.result) + return arglocs + + def prepare_op_new_with_vtable(self, op, fcond): + classint = op.getarg(0).getint() + descrsize = heaptracker.vtable2descr(self.assembler.cpu, classint) + arglocs = self._prepare_args_for_new_op(descrsize) + arglocs.append(imm(classint)) + return arglocs + + def prepare_op_new_array(self, op, fcond): + gc_ll_descr = self.assembler.cpu.gc_ll_descr + if gc_ll_descr.get_funcptr_for_newarray is not None: + raise NotImplementedError + # boehm GC + itemsize, scale, basesize, ofs_length, _ = ( + self._unpack_arraydescr(op.getdescr())) + return self._malloc_varsize(basesize, ofs_length, itemsize, op) + + def prepare_op_newstr(self, op, fcond): + gc_ll_descr = self.assembler.cpu.gc_ll_descr + if gc_ll_descr.get_funcptr_for_newstr is not None: + raise NotImplementedError + # boehm GC + ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, + self.assembler.cpu.translate_support_code) + assert itemsize == 1 + return self._malloc_varsize(ofs_items, ofs, itemsize, op) + + def prepare_op_newunicode(self, op, fcond): + gc_ll_descr = self.assembler.cpu.gc_ll_descr + if gc_ll_descr.get_funcptr_for_newunicode is not None: + raise NotImplementedError + # boehm GC + ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE, + self.assembler.cpu.translate_support_code) + _, itemsize, _ = symbolic.get_array_token(rstr.UNICODE, + self.assembler.cpu.translate_support_code) + return self._malloc_varsize(ofs_items, ofs, itemsize, op) + + def _malloc_varsize(self, ofs_items, ofs_length, itemsize, op): + v = op.getarg(0) + res_v = op.result + boxes = [v, res_v] + itemsize_box = ConstInt(itemsize) + ofs_items_box = ConstInt(ofs_items) + if self._check_imm_arg(ofs_items_box): + ofs_items_loc = self.convert_to_imm(ofs_items_box) + else: + ofs_items_loc, ofs_items_box = self._ensure_value_is_boxed(ofs_items_box, boxes) + boxes.append(ofs_items_box) + vloc, v = self._ensure_value_is_boxed(v, [res_v]) + boxes.append(v) + size, size_box = self._ensure_value_is_boxed(itemsize_box, boxes) + boxes.append(size_box) + self.assembler._regalloc_malloc_varsize(size, size_box, + vloc, ofs_items_loc, self, res_v) + base_loc = self.make_sure_var_in_reg(res_v) + value_loc = self.make_sure_var_in_reg(v) + self.possibly_free_vars(boxes) + assert value_loc.is_reg() + assert base_loc.is_reg() + return [value_loc, base_loc, imm(ofs_length)] + + prepare_op_cond_call_gc_wb = void + prepare_op_debug_merge_point = void + prepare_op_jit_debug = void + + def prepare_op_force_token(self, op, fcond): + res_loc = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) + return [res_loc] + + def prepare_guard_call_may_force(self, op, guard_op, fcond): + faildescr = guard_op.getdescr() + fail_index = self.assembler.cpu.get_fail_descr_number(faildescr) + # XXX remove tempbox when ip can be used in gen_load_int + t = TempBox() + l0 = self.force_allocate_reg(t) + self.possibly_free_var(t) + self.assembler._write_fail_index(fail_index, l0) + args = [rffi.cast(lltype.Signed, op.getarg(0).getint())] + # force all reg values to be spilled when calling + self.assembler.emit_op_call(op, args, self, fcond, spill_all_regs=True) + + return self._prepare_guard(guard_op) + + def prepare_guard_call_assembler(self, op, guard_op, fcond): + faildescr = guard_op.getdescr() + fail_index = self.assembler.cpu.get_fail_descr_number(faildescr) + # XXX remove tempbox when ip can be used in gen_load_int + t = TempBox() + l0 = self.force_allocate_reg(t) + self.possibly_free_var(t) + self.assembler._write_fail_index(fail_index, l0) + + def _prepare_args_for_new_op(self, new_args): + gc_ll_descr = self.assembler.cpu.gc_ll_descr + args = gc_ll_descr.args_for_new(new_args) + arglocs = [] + for i in range(len(args)): + arg = args[i] + t = TempBox() + l = self.force_allocate_reg(t, selected_reg=r.all_regs[i]) + self.assembler.load(l, imm(arg)) + arglocs.append(t) + return arglocs + + #XXX from ../x86/regalloc.py:791 + def _unpack_fielddescr(self, fielddescr): + assert isinstance(fielddescr, BaseFieldDescr) + ofs = fielddescr.offset + size = fielddescr.get_field_size(self.assembler.cpu.translate_support_code) + ptr = fielddescr.is_pointer_field() + return ofs, size, ptr + + #XXX from ../x86/regalloc.py:779 + def _unpack_arraydescr(self, arraydescr): + assert isinstance(arraydescr, BaseArrayDescr) + cpu = self.assembler.cpu + ofs_length = arraydescr.get_ofs_length(cpu.translate_support_code) + ofs = arraydescr.get_base_size(cpu.translate_support_code) + size = arraydescr.get_item_size(cpu.translate_support_code) + ptr = arraydescr.is_array_of_pointers() + scale = 0 + while (1 << scale) < size: + scale += 1 + assert (1 << scale) == size + return size, scale, ofs, ofs_length, ptr + + + +def make_operation_list(): + def notimplemented(self, op, fcond): + raise NotImplementedError, op + + operations = [None] * (rop._LAST+1) + for key, value in rop.__dict__.items(): + key = key.lower() + if key.startswith('_'): + continue + methname = 'prepare_op_%s' % key + if hasattr(ARMRegisterManager, methname): + func = getattr(ARMRegisterManager, methname).im_func + else: + func = notimplemented + operations[value] = func + return operations + +def make_guard_operation_list(): + def notimplemented(self, op, guard_op, fcond): + raise NotImplementedError, op + guard_operations = [notimplemented] * rop._LAST + for key, value in rop.__dict__.items(): + key = key.lower() + if key.startswith('_'): + continue + methname = 'prepare_guard_%s' % key + if hasattr(ARMRegisterManager, methname): + func = getattr(ARMRegisterManager, methname).im_func + guard_operations[value] = func + return guard_operations + +ARMRegisterManager.operations = make_operation_list() +ARMRegisterManager.operations_with_guard = make_guard_operation_list() From david at codespeak.net Sun Dec 12 14:05:51 2010 From: david at codespeak.net (david at codespeak.net) Date: Sun, 12 Dec 2010 14:05:51 +0100 (CET) Subject: [pypy-svn] r79999 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . helper Message-ID: <20101212130551.870DB282B90@codespeak.net> Author: david Date: Sun Dec 12 14:05:48 2010 New Revision: 79999 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Log: Improve constant loading, some related fixes and well and register allocation fixes Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Sun Dec 12 14:05:48 2010 @@ -375,9 +375,9 @@ if regalloc.frame_manager.frame_depth == 1: return n = (regalloc.frame_manager.frame_depth-1)*WORD - self._adjust_sp(n, regalloc, cb, base_reg=r.fp) + self._adjust_sp(n, cb, base_reg=r.fp) - def _adjust_sp(self, n, regalloc, cb=None, fcond=c.AL, base_reg=r.sp): + def _adjust_sp(self, n, cb=None, fcond=c.AL, base_reg=r.sp): if cb is None: cb = self.mc if n < 0: @@ -391,14 +391,11 @@ else: cb.SUB_ri(r.sp.value, base_reg.value, n) else: - b = TempBox() - reg = regalloc.force_allocate_reg(b) - cb.gen_load_int(reg.value, n, cond=fcond) + cb.gen_load_int(r.ip.value, n, cond=fcond) if rev: cb.ADD_rr(r.sp.value, base_reg.value, reg.value, cond=fcond) else: cb.SUB_rr(r.sp.value, base_reg.value, reg.value, cond=fcond) - regalloc.possibly_free_var(b) def _walk_operations(self, operations, regalloc): fcond=c.AL Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py Sun Dec 12 14:05:48 2010 @@ -120,18 +120,18 @@ def currpos(self): return self._pos - size_of_gen_load_int = 7 * WORD + size_of_gen_load_int = 4 * WORD + #XXX use MOV_ri if value fits in imm def gen_load_int(self, r, value, cond=cond.AL): """r is the register number, value is the value to be loaded to the register""" - assert r != reg.ip.value, 'ip is used to load int' - ip = reg.ip.value - self.MOV_ri(r, (value & 0xFF), cond=cond) - for offset in range(8, 25, 8): - t = (value >> offset) & 0xFF - self.MOV_ri(ip, t, cond=cond) - self.ORR_rr(r, r, ip, offset, cond=cond) + for offset, shift in zip(range(8, 25, 8), range(12, 0, -4)): + b = (value >> offset) & 0xFF + if b == 0: + continue + t = b | (shift << 8) + self.ORR_ri(r, r, imm=t, cond=cond) class ARMv7InMemoryBuilder(AbstractARMv7Builder): Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py Sun Dec 12 14:05:48 2010 @@ -49,10 +49,13 @@ arg2 = self.make_sure_var_in_reg(a1, selected_reg=r.r1) assert arg1 == r.r0 assert arg2 == r.r1 + spilled = False if isinstance(a0, Box) and self.stays_alive(a0): + spilled = True self.force_spill_var(a0) self.after_call(op.result) - self.possibly_free_var(a0) + if spilled: + self.possibly_free_var(a0) self.possibly_free_var(a1) self.possibly_free_var(op.result) return [] Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Sun Dec 12 14:05:48 2010 @@ -280,7 +280,7 @@ if n_args > 4: stack_args = n_args - 4 n = stack_args*WORD - self._adjust_sp(n, regalloc, fcond=fcond) + self._adjust_sp(n, fcond=fcond) for i in range(4, n_args): reg, box = regalloc._ensure_value_is_boxed(args[i], regalloc) self.mc.STR_ri(reg.value, r.sp.value, (i-4)*WORD) @@ -293,7 +293,7 @@ # readjust the sp in case we passed some args on the stack if n_args > 4: assert n > 0 - self._adjust_sp(-n, regalloc, fcond=fcond) + self._adjust_sp(-n, fcond=fcond) # restore the argumets stored on the stack if spill_all_regs: @@ -653,12 +653,10 @@ assert value <= 0xff # check value - t = TempBox() - #XXX is this correct? + loc, t = regalloc._ensure_value_is_boxed(ConstInt(value)) resloc = regalloc.force_allocate_reg(resbox) - loc = regalloc.force_allocate_reg(t) - self.mc.gen_load_int(loc.value, value) - self.mc.CMP_rr(resloc.value, loc.value) + self.mc.gen_load_int(r.ip.value, value) + self.mc.CMP_rr(resloc.value, r.ip.value) regalloc.possibly_free_var(resbox) fast_jmp_pos = self.mc.currpos() @@ -702,8 +700,7 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] - loc = regalloc.force_allocate_reg(t) - resloc = regalloc.force_allocate_reg(op.result, [t]) + resloc = regalloc.force_allocate_reg(op.result) kind = op.result.type if kind == INT: adr = self.fail_boxes_int.get_addr_for_num(0) @@ -711,9 +708,8 @@ adr = self.fail_boxes_ptr.get_addr_for_num(0) else: raise AssertionError(kind) - self.mc.gen_load_int(loc.value, adr) - self.mc.LDR_ri(resloc.value, loc.value) - regalloc.possibly_free_var(t) + self.mc.gen_load_int(r.ip.value, adr) + self.mc.LDR_ri(resloc.value, r.ip.value) offset = self.mc.currpos() - jmp_pos pmc = ARMv7InMemoryBuilder(jmp_location, WORD) @@ -737,9 +733,9 @@ self._emit_guard(guard_op, arglocs, c.GE) return fcond - def _write_fail_index(self, fail_index, temp_reg): - self.mc.gen_load_int(temp_reg.value, fail_index) - self.mc.STR_ri(temp_reg.value, r.fp.value) + def _write_fail_index(self, fail_index): + self.mc.gen_load_int(r.ip.value, fail_index) + self.mc.STR_ri(r.ip.value, r.fp.value) class AllocOpAssembler(object): @@ -771,23 +767,17 @@ callargs = arglocs[:-1] self._emit_call(self.malloc_func_addr, callargs, regalloc, result=op.result) - self.set_vtable(op.result, classint, regalloc) + self.set_vtable(op.result, classint) #XXX free args here, because _emit_call works on regalloc regalloc.possibly_free_vars(callargs) regalloc.possibly_free_var(op.result) return fcond - def set_vtable(self, box, vtable, regalloc): + def set_vtable(self, box, vtable): if self.cpu.vtable_offset is not None: - loc = regalloc.loc(box) # r0 - assert loc is r.r0 adr = rffi.cast(lltype.Signed, vtable) - t = TempBox() - loc_vtable = regalloc.force_allocate_reg(t, [box]) - assert loc_vtable is not loc - regalloc.possibly_free_var(t) - self.mc.gen_load_int(loc_vtable.value, adr) - self.mc.STR_ri(loc_vtable.value, loc.value, self.cpu.vtable_offset) + self.mc.gen_load_int(r.ip.value, adr) + self.mc.STR_ri(r.ip.value, r.r0.value, self.cpu.vtable_offset) def emit_op_new_array(self, op, arglocs, regalloc, fcond): value_loc, base_loc, ofs_length = arglocs Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Sun Dec 12 14:05:48 2010 @@ -79,28 +79,14 @@ # is also used on op args, which is a non-resizable list self.possibly_free_vars(list(inputargs)) - #XXX remove - def force_allocate_reg(self, v, forbidden_vars=[], selected_reg=None, - need_lower_byte=False): - # override ../llsupport/regalloc.py to set longevity for vals not in longevity - self._check_type(v) - if isinstance(v, TempBox) or v not in self.longevity: - self.longevity[v] = (self.position, self.position) - loc = self.try_allocate_reg(v, selected_reg, - need_lower_byte=need_lower_byte) - if loc: - return loc - loc = self._spill_var(v, forbidden_vars, selected_reg, - need_lower_byte=need_lower_byte) - prev_loc = self.reg_bindings.get(v, None) - if prev_loc is not None: - self.free_regs.append(prev_loc) - self.reg_bindings[v] = loc - return loc - def force_spill_var(self, var): self._sync_var(var) - del self.reg_bindings[var] + try: + loc = self.reg_bindings[var] + del self.reg_bindings[var] + self.free_regs.append(loc) + except KeyError: + import pdb; pdb.set_trace() def _check_imm_arg(self, arg, size=0xFF, allow_zero=True): if isinstance(arg, ConstInt): @@ -112,8 +98,6 @@ return False def _ensure_value_is_boxed(self, thing, forbidden_vars=[]): - # XXX create TempBox subclasses with the corresponding type flags and - # remove the overridden force_allocate_reg once done box = None loc = None if isinstance(thing, Const): @@ -207,13 +191,13 @@ boxes.append(box) reg2, box = self._ensure_value_is_boxed(a1,forbidden_vars=boxes) boxes.append(box) - self.possibly_free_vars(boxes) res = self.force_allocate_reg(op.result, boxes) args.append(reg1) args.append(reg2) args.append(res) args = self._prepare_guard(guard, args) + self.possibly_free_vars(boxes) self.possibly_free_var(op.result) return args @@ -305,9 +289,10 @@ boxes.append(box) else: l1 = self.make_sure_var_in_reg(a1) - self.possibly_free_vars(boxes) assert op.result is None - return self._prepare_guard(op, [l0, l1]) + arglocs = self._prepare_guard(op, [l0, l1]) + self.possibly_free_vars(boxes) + return arglocs def prepare_op_guard_no_overflow(self, op, fcond): return self._prepare_guard(op) @@ -327,16 +312,19 @@ boxes.append(resloc) else: resloc = None - self.possibly_free_vars(boxes) + # There is some redundancy here ?! pos_exc_value = imm(self.assembler.cpu.pos_exc_value()) pos_exception = imm(self.assembler.cpu.pos_exception()) - return self._prepare_guard(op, [loc, loc1, resloc, pos_exc_value, pos_exception]) + arglocs = self._prepare_guard(op, [loc, loc1, resloc, pos_exc_value, pos_exception]) + self.possibly_free_vars(boxes) + return arglocs def prepare_op_guard_no_exception(self, op, fcond): loc, box = self._ensure_value_is_boxed( ConstInt(self.assembler.cpu.pos_exception())) + arglocs = self._prepare_guard(op, [loc]) self.possibly_free_var(box) - return self._prepare_guard(op, [loc]) + return arglocs def prepare_op_guard_class(self, op, fcond): return self._prepare_guard_class(op, fcond) @@ -356,8 +344,9 @@ y_val = rffi.cast(lltype.Signed, op.getarg(1).getint()) self.assembler.load(y, imm(y_val)) + arglocs = self._prepare_guard(op, [x, y]) self.possibly_free_vars(boxes) - return self._prepare_guard(op, [x, y]) + return arglocs def prepare_op_jump(self, op, fcond): descr = op.getdescr() @@ -645,11 +634,7 @@ def prepare_guard_call_may_force(self, op, guard_op, fcond): faildescr = guard_op.getdescr() fail_index = self.assembler.cpu.get_fail_descr_number(faildescr) - # XXX remove tempbox when ip can be used in gen_load_int - t = TempBox() - l0 = self.force_allocate_reg(t) - self.possibly_free_var(t) - self.assembler._write_fail_index(fail_index, l0) + self.assembler._write_fail_index(fail_index) args = [rffi.cast(lltype.Signed, op.getarg(0).getint())] # force all reg values to be spilled when calling self.assembler.emit_op_call(op, args, self, fcond, spill_all_regs=True) @@ -659,11 +644,7 @@ def prepare_guard_call_assembler(self, op, guard_op, fcond): faildescr = guard_op.getdescr() fail_index = self.assembler.cpu.get_fail_descr_number(faildescr) - # XXX remove tempbox when ip can be used in gen_load_int - t = TempBox() - l0 = self.force_allocate_reg(t) - self.possibly_free_var(t) - self.assembler._write_fail_index(fail_index, l0) + self.assembler._write_fail_index(fail_index) def _prepare_args_for_new_op(self, new_args): gc_ll_descr = self.assembler.cpu.gc_ll_descr From cfbolz at codespeak.net Sun Dec 12 14:24:54 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Sun, 12 Dec 2010 14:24:54 +0100 (CET) Subject: [pypy-svn] r80000 - pypy/trunk/pypy/rpython/lltypesystem Message-ID: <20101212132454.2E790282B90@codespeak.net> Author: cfbolz Date: Sun Dec 12 14:24:51 2010 New Revision: 80000 Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py Log: use full width of a signed even on 64 bit machines to make string finding probably a bit faster there Modified: pypy/trunk/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/rstr.py Sun Dec 12 14:24:51 2010 @@ -242,10 +242,7 @@ FAST_RFIND = 2 -# XXX: This should be set to the number of bits in a long. Having a lower -# value here doesn't break anything, it just decreases the accuracy of the -# bloom filter heuristic, which results in a worse runtime (but correct results) -BLOOM_WIDTH = 32 +from pypy.rlib.rarithmetic import LONG_BIT as BLOOM_WIDTH def bloom_add(mask, c): From arigo at codespeak.net Sun Dec 12 17:12:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 12 Dec 2010 17:12:24 +0100 (CET) Subject: [pypy-svn] r80001 - pypy/extradoc/planning Message-ID: <20101212161224.8D867282B90@codespeak.net> Author: arigo Date: Sun Dec 12 17:12:22 2010 New Revision: 80001 Modified: pypy/extradoc/planning/1.4.1.txt Log: Update with what I see as the list of tasks really needed for 1.4.1. Modified: pypy/extradoc/planning/1.4.1.txt ============================================================================== --- pypy/extradoc/planning/1.4.1.txt (original) +++ pypy/extradoc/planning/1.4.1.txt Sun Dec 12 17:12:22 2010 @@ -24,8 +24,12 @@ Plan: -* Out of line guards (?) +* Out of line guards (?) -> after the release -* Merge jit-unroll-loops (?) +* Merge jit-unroll-loops (?) -> after the release -* Migrate to mercurial +* Migrate to mercurial -> should not be a blocker + +* Understand what's going on on Windows with pypy-c translate.py -Ojit + +* finish the removal of sys.setrecursionlimit() From arigo at codespeak.net Sun Dec 12 17:15:24 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 12 Dec 2010 17:15:24 +0100 (CET) Subject: [pypy-svn] r80002 - pypy/branch/jit-stackcheck Message-ID: <20101212161524.A4199282B90@codespeak.net> Author: arigo Date: Sun Dec 12 17:15:23 2010 New Revision: 80002 Added: pypy/branch/jit-stackcheck/ - copied from r80001, pypy/trunk/ Log: A branch to implement stack checks in the jit. From arigo at codespeak.net Sun Dec 12 18:09:16 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 12 Dec 2010 18:09:16 +0100 (CET) Subject: [pypy-svn] r80003 - in pypy/trunk/pypy/jit/backend: llsupport llsupport/test x86 x86/tool Message-ID: <20101212170916.13434282B90@codespeak.net> Author: arigo Date: Sun Dec 12 18:09:14 2010 New Revision: 80003 Modified: pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py pypy/trunk/pypy/jit/backend/x86/codebuf.py pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py Log: Reintroduce the possibility to dump machine code: it goes now to the standard log file. Use PYPYLOG=jit-backend-dump:log. Load it with jit/backend/x86/tool/viewcode.py, by calling it with the 'log' filename. Modified: pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/asmmemmgr.py Sun Dec 12 18:09:14 2010 @@ -2,6 +2,8 @@ from pypy.rlib.rarithmetic import intmask, r_uint, LONG_BIT from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rmmap +from pypy.rlib.debug import debug_start, debug_print, debug_stop +from pypy.rlib.debug import have_debug_prints from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -265,6 +267,31 @@ targetindex -= self.SUBBLOCK_SIZE assert not block + def _dump(self, addr, logname, backend=None): + debug_start(logname) + if have_debug_prints(): + # + if backend is not None: + debug_print('BACKEND', backend) + # + from pypy.jit.backend.hlinfo import highleveljitinfo + if highleveljitinfo.sys_executable: + debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable) + # + HEX = '0123456789ABCDEF' + dump = [] + src = rffi.cast(rffi.CCHARP, addr) + for p in range(self.get_relative_pos()): + o = ord(src[p]) + dump.append(HEX[o >> 4]) + dump.append(HEX[o & 15]) + debug_print('CODE_DUMP', + '@%x' % addr, + '+0 ', # backwards compatibility + ''.join(dump)) + # + debug_stop(logname) + def materialize(self, asmmemmgr, allblocks, gcrootmap=None): size = self.get_relative_pos() malloced = asmmemmgr.malloc(size, size) Modified: pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/test/test_asmmemmgr.py Sun Dec 12 18:09:14 2010 @@ -3,6 +3,7 @@ from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib import debug def test_get_index(): @@ -184,9 +185,11 @@ def test_blockbuildermixin(translated=True): mc = BlockBuilderMixin(translated) + writtencode = [] for i in range(mc.SUBBLOCK_SIZE * 2 + 3): assert mc.get_relative_pos() == i mc.writechar(chr(i % 255)) + writtencode.append(chr(i % 255)) if translated: assert mc._cursubindex == 3 assert mc._cursubblock @@ -196,16 +199,26 @@ # for i in range(0, mc.SUBBLOCK_SIZE * 2 + 3, 2): mc.overwrite(i, chr((i + 63) % 255)) + writtencode[i] = chr((i + 63) % 255) # p = lltype.malloc(rffi.CCHARP.TO, mc.SUBBLOCK_SIZE * 2 + 3, flavor='raw') addr = rffi.cast(lltype.Signed, p) mc.copy_to_raw_memory(addr) # for i in range(mc.SUBBLOCK_SIZE * 2 + 3): - if i & 1: - assert p[i] == chr(i % 255) - else: - assert p[i] == chr((i + 63) % 255) + assert p[i] == writtencode[i] + # + debug._log = debug.DebugLog() + try: + mc._dump(addr, 'test-logname-section') + log = list(debug._log) + finally: + debug._log = None + encoded = ''.join(writtencode).encode('hex').upper() + ataddr = '@%x' % addr + assert log == [('test-logname-section', + [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])] + # lltype.free(p, flavor='raw') def test_blockbuildermixin2(): Modified: pypy/trunk/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/trunk/pypy/jit/backend/x86/codebuf.py Sun Dec 12 18:09:14 2010 @@ -10,8 +10,10 @@ # like this if IS_X86_32: codebuilder_cls = X86_32_CodeBuilder + backend_name = 'x86' elif IS_X86_64: codebuilder_cls = X86_64_CodeBuilder + backend_name = 'x86_64' class MachineCodeBlockWrapper(BlockBuilderMixin, @@ -34,3 +36,4 @@ adr = rffi.cast(rffi.LONGP, p - WORD) adr[0] = intmask(adr[0] - p) valgrind.discard_translations(addr, self.get_relative_pos()) + self._dump(addr, "jit-backend-dump", backend_name) Modified: pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/trunk/pypy/jit/backend/x86/tool/viewcode.py Sun Dec 12 18:09:14 2010 @@ -1,11 +1,11 @@ #! /usr/bin/env python """ -Viewer for the CODE_DUMP output of compiled programs generating code. +Viewer for the output of compiled programs generating code. +Use on the log files created with 'PYPYLOG=jit-backend-dump:log'. Try: - ./viewcode.py dumpfile.txt -or - /tmp/usession-xxx/testing_1/testing_1 -var 4 2>&1 | ./viewcode.py + ./viewcode.py --text log # text only disassembly + ./viewcode.py log # also includes a pygame viewer """ import autopath @@ -179,6 +179,7 @@ self.symbols = {} self.logentries = {} self.backend_name = None + self.executable_name = None def parse(self, f, textonly=True): for line in f: @@ -214,7 +215,9 @@ self.logentries[addr] = pieces[3] elif line.startswith('SYS_EXECUTABLE '): filename = line[len('SYS_EXECUTABLE '):].strip() - self.symbols.update(load_symbols(filename)) + if filename != self.executable_name: + self.symbols.update(load_symbols(filename)) + self.executable_name = filename def find_cross_references(self): # find cross-references between blocks @@ -375,10 +378,19 @@ showgraph = False else: showgraph = True - if len(sys.argv) == 1: - f = sys.stdin - else: - f = open(sys.argv[1], 'r') + if len(sys.argv) != 2: + print >> sys.stderr, __doc__ + sys.exit(2) + # + import cStringIO + from pypy.tool import logparser + log1 = logparser.parse_log_file(sys.argv[1]) + text1 = logparser.extract_category(log1, catprefix='jit-backend-dump') + f = cStringIO.StringIO() + f.writelines(text1) + f.seek(0) + del log1, text1 + # world = World() world.parse(f) if showgraph: From afa at codespeak.net Sun Dec 12 18:11:25 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 12 Dec 2010 18:11:25 +0100 (CET) Subject: [pypy-svn] r80004 - in pypy/branch/fast-forward: lib-python/modified-2.5.2/encodings lib-python/modified-2.5.2/test lib-python/modified-2.5.2/test/output lib_pypy/_ctypes pypy/config pypy/doc/config pypy/interpreter pypy/interpreter/pyparser pypy/interpreter/pyparser/test pypy/interpreter/test pypy/jit/backend/llsupport pypy/jit/metainterp pypy/jit/metainterp/optimizeopt pypy/jit/metainterp/test pypy/module/_lsprof pypy/module/_lsprof/test pypy/module/cpyext pypy/module/cpyext/src pypy/module/exceptions pypy/module/imp pypy/module/posix pypy/module/posix/test pypy/module/select pypy/module/sys pypy/module/test_lib_pypy/ctypes_tests pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rpython/lltypesystem pypy/rpython/lltypesystem/test pypy/rpython/memory/gc pypy/rpython/memory/gctransform pypy/rpython/module pypy/translator/c/test pypy/translator/goal pypy/translator/sandbox/test Message-ID: <20101212171125.18BBA282B90@codespeak.net> Author: afa Date: Sun Dec 12 18:11:19 2010 New Revision: 80004 Added: pypy/branch/fast-forward/lib-python/modified-2.5.2/test/test_eof.py - copied unchanged from r80002, pypy/trunk/lib-python/modified-2.5.2/test/test_eof.py pypy/branch/fast-forward/pypy/doc/config/translation.jit_ffi.txt - copied unchanged from r80002, pypy/trunk/pypy/doc/config/translation.jit_ffi.txt pypy/branch/fast-forward/pypy/module/cpyext/pypyintf.py - copied unchanged from r80002, pypy/trunk/pypy/module/cpyext/pypyintf.py pypy/branch/fast-forward/pypy/translator/goal/targetsha1sum.py - copied unchanged from r80002, pypy/trunk/pypy/translator/goal/targetsha1sum.py Removed: pypy/branch/fast-forward/lib-python/modified-2.5.2/encodings/ Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/test/output/test_cProfile pypy/branch/fast-forward/lib-python/modified-2.5.2/test/test_genexps.py pypy/branch/fast-forward/lib_pypy/_ctypes/primitive.py pypy/branch/fast-forward/pypy/config/pypyoption.py pypy/branch/fast-forward/pypy/config/translationoption.py pypy/branch/fast-forward/pypy/interpreter/baseobjspace.py pypy/branch/fast-forward/pypy/interpreter/executioncontext.py pypy/branch/fast-forward/pypy/interpreter/function.py pypy/branch/fast-forward/pypy/interpreter/mixedmodule.py pypy/branch/fast-forward/pypy/interpreter/pyparser/error.py pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenizer.py pypy/branch/fast-forward/pypy/interpreter/pyparser/test/test_pyparse.py pypy/branch/fast-forward/pypy/interpreter/test/test_compiler.py pypy/branch/fast-forward/pypy/interpreter/test/test_executioncontext.py pypy/branch/fast-forward/pypy/interpreter/test/test_function.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py pypy/branch/fast-forward/pypy/jit/metainterp/compile.py pypy/branch/fast-forward/pypy/jit/metainterp/graphpage.py pypy/branch/fast-forward/pypy/jit/metainterp/optimizeopt/__init__.py pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_fficall.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizefficall.py pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizeopt.py pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py pypy/branch/fast-forward/pypy/module/_lsprof/interp_lsprof.py pypy/branch/fast-forward/pypy/module/_lsprof/test/test_cprofile.py pypy/branch/fast-forward/pypy/module/cpyext/__init__.py pypy/branch/fast-forward/pypy/module/cpyext/src/getargs.c pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py pypy/branch/fast-forward/pypy/module/imp/importing.py pypy/branch/fast-forward/pypy/module/posix/__init__.py pypy/branch/fast-forward/pypy/module/posix/interp_posix.py pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py pypy/branch/fast-forward/pypy/module/select/interp_select.py pypy/branch/fast-forward/pypy/module/sys/vm.py pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py pypy/branch/fast-forward/pypy/objspace/std/complexobject.py pypy/branch/fast-forward/pypy/objspace/std/complextype.py pypy/branch/fast-forward/pypy/objspace/std/floattype.py pypy/branch/fast-forward/pypy/objspace/std/longobject.py pypy/branch/fast-forward/pypy/objspace/std/strutil.py pypy/branch/fast-forward/pypy/objspace/std/test/test_complexobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_strutil.py pypy/branch/fast-forward/pypy/rlib/rmd5.py pypy/branch/fast-forward/pypy/rlib/rmmap.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/llarena.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/lltype.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/rstr.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_llarena.py pypy/branch/fast-forward/pypy/rpython/memory/gc/base.py pypy/branch/fast-forward/pypy/rpython/memory/gc/env.py pypy/branch/fast-forward/pypy/rpython/memory/gc/minimark.py pypy/branch/fast-forward/pypy/rpython/memory/gctransform/asmgcroot.py pypy/branch/fast-forward/pypy/rpython/memory/gctransform/framework.py pypy/branch/fast-forward/pypy/rpython/module/ll_os.py pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py pypy/branch/fast-forward/pypy/translator/sandbox/test/test_sandbox.py Log: Merge from trunk: svn merge -r79849:80002 ../trunk Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/test/output/test_cProfile ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/test/output/test_cProfile (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/test/output/test_cProfile Sun Dec 12 18:11:19 2010 @@ -14,66 +14,66 @@ 4 0.116 0.029 0.120 0.030 test_cProfile.py:78(helper1) 2 0.000 0.000 0.140 0.070 test_cProfile.py:89(helper2_indirect) 8 0.312 0.039 0.400 0.050 test_cProfile.py:93(helper2) - 4 0.000 0.000 0.000 0.000 {append} - 1 0.000 0.000 0.000 0.000 {disable} 12 0.000 0.000 0.012 0.001 {hasattr} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} + 1 0.000 0.000 0.000 0.000 {method 'disable' of 'Profile' objects} 8 0.000 0.000 0.000 0.000 {range} 4 0.000 0.000 0.000 0.000 {sys.exc_info} Ordered by: standard name -Function called... - ncalls tottime cumtime -:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) -test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) - 8 0.000 0.000 {range} -test_cProfile.py:115(__getattr__) -> -test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) - 2 0.040 0.600 test_cProfile.py:60(helper) -test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 20 0.020 0.020 test_cProfile.py:53(mul) -test_cProfile.py:53(mul) -> -test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) - 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) - 6 0.234 0.300 test_cProfile.py:93(helper2) -test_cProfile.py:78(helper1) -> 4 0.000 0.000 {append} - 4 0.000 0.004 {hasattr} - 4 0.000 0.000 {sys.exc_info} -test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) - 2 0.078 0.100 test_cProfile.py:93(helper2) -test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) - 8 0.000 0.008 {hasattr} -{append} -> -{disable} -> -{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) -{range} -> -{sys.exc_info} -> +Function called... + ncalls tottime cumtime +:1() -> 1 0.270 1.000 test_cProfile.py:30(testfunc) +test_cProfile.py:103(subhelper) -> 16 0.016 0.016 test_cProfile.py:115(__getattr__) + 8 0.000 0.000 {range} +test_cProfile.py:115(__getattr__) -> +test_cProfile.py:30(testfunc) -> 1 0.014 0.130 test_cProfile.py:40(factorial) + 2 0.040 0.600 test_cProfile.py:60(helper) +test_cProfile.py:40(factorial) -> 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 20 0.020 0.020 test_cProfile.py:53(mul) +test_cProfile.py:53(mul) -> +test_cProfile.py:60(helper) -> 4 0.116 0.120 test_cProfile.py:78(helper1) + 2 0.000 0.140 test_cProfile.py:89(helper2_indirect) + 6 0.234 0.300 test_cProfile.py:93(helper2) +test_cProfile.py:78(helper1) -> 4 0.000 0.004 {hasattr} + 4 0.000 0.000 {method 'append' of 'list' objects} + 4 0.000 0.000 {sys.exc_info} +test_cProfile.py:89(helper2_indirect) -> 2 0.006 0.040 test_cProfile.py:40(factorial) + 2 0.078 0.100 test_cProfile.py:93(helper2) +test_cProfile.py:93(helper2) -> 8 0.064 0.080 test_cProfile.py:103(subhelper) + 8 0.000 0.008 {hasattr} +{hasattr} -> 12 0.012 0.012 test_cProfile.py:115(__getattr__) +{method 'append' of 'list' objects} -> +{method 'disable' of 'Profile' objects} -> +{range} -> +{sys.exc_info} -> Ordered by: standard name -Function was called by... - ncalls tottime cumtime -:1() <- -test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) -test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) - 12 0.012 0.012 {hasattr} -test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() -test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) - 20/3 0.130 0.147 test_cProfile.py:40(factorial) - 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) -test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) -test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) -test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) -test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) -test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) - 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) -{append} <- 4 0.000 0.000 test_cProfile.py:78(helper1) -{disable} <- -{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) - 8 0.000 0.008 test_cProfile.py:93(helper2) -{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) -{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +Function was called by... + ncalls tottime cumtime +:1() <- +test_cProfile.py:103(subhelper) <- 8 0.064 0.080 test_cProfile.py:93(helper2) +test_cProfile.py:115(__getattr__) <- 16 0.016 0.016 test_cProfile.py:103(subhelper) + 12 0.012 0.012 {hasattr} +test_cProfile.py:30(testfunc) <- 1 0.270 1.000 :1() +test_cProfile.py:40(factorial) <- 1 0.014 0.130 test_cProfile.py:30(testfunc) + 20/3 0.130 0.147 test_cProfile.py:40(factorial) + 2 0.006 0.040 test_cProfile.py:89(helper2_indirect) +test_cProfile.py:53(mul) <- 20 0.020 0.020 test_cProfile.py:40(factorial) +test_cProfile.py:60(helper) <- 2 0.040 0.600 test_cProfile.py:30(testfunc) +test_cProfile.py:78(helper1) <- 4 0.116 0.120 test_cProfile.py:60(helper) +test_cProfile.py:89(helper2_indirect) <- 2 0.000 0.140 test_cProfile.py:60(helper) +test_cProfile.py:93(helper2) <- 6 0.234 0.300 test_cProfile.py:60(helper) + 2 0.078 0.100 test_cProfile.py:89(helper2_indirect) +{hasattr} <- 4 0.000 0.004 test_cProfile.py:78(helper1) + 8 0.000 0.008 test_cProfile.py:93(helper2) +{method 'append' of 'list' objects} <- 4 0.000 0.000 test_cProfile.py:78(helper1) +{method 'disable' of 'Profile' objects} <- +{range} <- 8 0.000 0.000 test_cProfile.py:103(subhelper) +{sys.exc_info} <- 4 0.000 0.000 test_cProfile.py:78(helper1) Modified: pypy/branch/fast-forward/lib-python/modified-2.5.2/test/test_genexps.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.5.2/test/test_genexps.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.5.2/test/test_genexps.py Sun Dec 12 18:11:19 2010 @@ -77,10 +77,10 @@ Verify that parenthesis are required in a statement >>> def f(n): - ... return i*i for i in xrange(n) + ... return i*i for i in xrange(n) #doctest: +ELLIPSIS Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: invalid syntax... Verify that parenthesis are required when used as a keyword argument value Modified: pypy/branch/fast-forward/lib_pypy/_ctypes/primitive.py ============================================================================== --- pypy/branch/fast-forward/lib_pypy/_ctypes/primitive.py (original) +++ pypy/branch/fast-forward/lib_pypy/_ctypes/primitive.py Sun Dec 12 18:11:19 2010 @@ -295,9 +295,9 @@ self.value = value def _ensure_objects(self): - if self._type_ in 'zZ': - return self._objects - return None + if self._type_ not in 'zZP': + assert self._objects is None + return self._objects def _getvalue(self): return self._buffer[0] Modified: pypy/branch/fast-forward/pypy/config/pypyoption.py ============================================================================== --- pypy/branch/fast-forward/pypy/config/pypyoption.py (original) +++ pypy/branch/fast-forward/pypy/config/pypyoption.py Sun Dec 12 18:11:19 2010 @@ -78,6 +78,7 @@ "_rawffi": [("objspace.usemodules.struct", True)], "cpyext": [("translation.secondaryentrypoints", "cpyext"), ("translation.shared", sys.platform == "win32")], + "_ffi": [("translation.jit_ffi", True)], } module_import_dependencies = { Modified: pypy/branch/fast-forward/pypy/config/translationoption.py ============================================================================== --- pypy/branch/fast-forward/pypy/config/translationoption.py (original) +++ pypy/branch/fast-forward/pypy/config/translationoption.py Sun Dec 12 18:11:19 2010 @@ -117,6 +117,7 @@ ChoiceOption("jit_profiler", "integrate profiler support into the JIT", ["off", "oprofile"], default="off"), + BoolOption("jit_ffi", "optimize libffi calls", default=False), # misc BoolOption("verbose", "Print extra information", default=False), Modified: pypy/branch/fast-forward/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/baseobjspace.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/baseobjspace.py Sun Dec 12 18:11:19 2010 @@ -865,14 +865,14 @@ def call_args_and_c_profile(self, frame, w_func, args): ec = self.getexecutioncontext() - ec.c_call_trace(frame, w_func) + ec.c_call_trace(frame, w_func, args) try: w_res = self.call_args(w_func, args) except OperationError, e: w_value = e.get_w_value(self) ec.c_exception_trace(frame, w_value) raise - ec.c_return_trace(frame, w_func) + ec.c_return_trace(frame, w_func, args) return w_res def call_method(self, w_obj, methname, *arg_w): @@ -1182,6 +1182,27 @@ self.wrap("expected a 32-bit integer")) return value + def c_filedescriptor_w(self, w_fd): + try: + fd = self.c_int_w(w_fd) + except OperationError, e: + if not e.match(self, self.w_TypeError): + raise + try: + w_fileno = self.getattr(w_fd, self.wrap('fileno')) + except OperationError, e: + if e.match(self, self.w_AttributeError): + raise OperationError(self.w_TypeError, + self.wrap("argument must be an int, " + "or have a fileno() method.")) + raise + w_fd = self.call_function(w_fileno) + fd = self.c_int_w(w_fd) + if fd < 0: + raise operationerrfmt(self.w_ValueError, + "file descriptor cannot be a negative integer (%d)", fd) + return fd + def warn(self, msg, w_warningcls): self.appexec([self.wrap(msg), w_warningcls], """(msg, warningcls): import warnings Modified: pypy/branch/fast-forward/pypy/interpreter/executioncontext.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/executioncontext.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/executioncontext.py Sun Dec 12 18:11:19 2010 @@ -123,19 +123,30 @@ return lst # coroutine: I think this is all, folks! - def c_call_trace(self, frame, w_func): + def c_call_trace(self, frame, w_func, args=None): "Profile the call of a builtin function" - if self.profilefunc is None: - frame.is_being_profiled = False - else: - self._trace(frame, 'c_call', w_func) + self._c_call_return_trace(frame, w_func, args, 'c_call') - def c_return_trace(self, frame, w_retval): + def c_return_trace(self, frame, w_func, args=None): "Profile the return from a builtin function" + self._c_call_return_trace(frame, w_func, args, 'c_return') + + def _c_call_return_trace(self, frame, w_func, args, event): if self.profilefunc is None: frame.is_being_profiled = False else: - self._trace(frame, 'c_return', w_retval) + # undo the effect of the CALL_METHOD bytecode, which would be + # that even on a built-in method call like '[].append()', + # w_func is actually the unbound function 'append'. + from pypy.interpreter.function import FunctionWithFixedCode + if isinstance(w_func, FunctionWithFixedCode) and args is not None: + w_firstarg = args.firstarg() + if w_firstarg is not None: + from pypy.interpreter.function import descr_function_get + w_func = descr_function_get(self.space, w_func, w_firstarg, + self.space.type(w_firstarg)) + # + self._trace(frame, event, w_func) def c_exception_trace(self, frame, w_exc): "Profile function called upon OperationError." Modified: pypy/branch/fast-forward/pypy/interpreter/function.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/function.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/function.py Sun Dec 12 18:11:19 2010 @@ -48,7 +48,7 @@ return "<%s %s>" % (self.__class__.__name__, name) def call_args(self, args): - # delegate activation to code + # delegate activation to code return self.getcode().funcrun(self, args) def call_obj_args(self, w_obj, args): @@ -61,17 +61,17 @@ return _get_immutable_code(self) return jit.hint(self.code, promote=True) return self.code - + def funccall(self, *args_w): # speed hack from pypy.interpreter import gateway from pypy.interpreter.pycode import PyCode - + code = self.getcode() # hook for the jit nargs = len(args_w) fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: if nargs == 0: - assert isinstance(code, gateway.BuiltinCode0) + assert isinstance(code, gateway.BuiltinCode0) return code.fastcall_0(self.space, self) elif nargs == 1: assert isinstance(code, gateway.BuiltinCode1) @@ -80,22 +80,22 @@ assert isinstance(code, gateway.BuiltinCode2) return code.fastcall_2(self.space, self, args_w[0], args_w[1]) elif nargs == 3: - assert isinstance(code, gateway.BuiltinCode3) + assert isinstance(code, gateway.BuiltinCode3) return code.fastcall_3(self.space, self, args_w[0], args_w[1], args_w[2]) elif nargs == 4: - assert isinstance(code, gateway.BuiltinCode4) + assert isinstance(code, gateway.BuiltinCode4) return code.fastcall_4(self.space, self, args_w[0], args_w[1], args_w[2], args_w[3]) elif (nargs|PyCode.FLATPYCALL) == fast_natural_arity: - assert isinstance(code, PyCode) + assert isinstance(code, PyCode) if nargs < 5: new_frame = self.space.createframe(code, self.w_func_globals, self.closure) for i in funccallunrolling: if i < nargs: new_frame.fastlocals_w[i] = args_w[i] - return new_frame.run() + return new_frame.run() elif nargs >= 1 and fast_natural_arity == Code.PASSTHROUGHARGS1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) return code.funcrun_obj(self, args_w[0], @@ -106,9 +106,9 @@ def funccall_valuestack(self, nargs, frame): # speed hack from pypy.interpreter import gateway from pypy.interpreter.pycode import PyCode - + code = self.getcode() # hook for the jit - fast_natural_arity = code.fast_natural_arity + fast_natural_arity = code.fast_natural_arity if nargs == fast_natural_arity: if nargs == 0: assert isinstance(code, gateway.BuiltinCode0) @@ -143,7 +143,7 @@ w_obj = frame.peekvalue(nargs-1) args = frame.make_arguments(nargs-1) return code.funcrun_obj(self, w_obj, args) - + args = frame.make_arguments(nargs) return self.call_args(args) @@ -155,8 +155,8 @@ for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) new_frame.fastlocals_w[i] = w_arg - - return new_frame.run() + + return new_frame.run() @jit.unroll_safe def _flat_pycall_defaults(self, code, nargs, frame, defs_to_load): @@ -166,7 +166,7 @@ for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) new_frame.fastlocals_w[i] = w_arg - + defs_w = self.defs_w ndefs = len(defs_w) start = ndefs-defs_to_load @@ -174,7 +174,7 @@ for j in xrange(start, ndefs): new_frame.fastlocals_w[i] = defs_w[j] i += 1 - return new_frame.run() + return new_frame.run() def getdict(self): if self.w_func_dict is None: @@ -188,7 +188,7 @@ # unwrapping is done through unwrap_specs in typedef.py - def descr_function__new__(space, w_subtype, w_code, w_globals, + def descr_function__new__(space, w_subtype, w_code, w_globals, w_name=None, w_argdefs=None, w_closure=None): code = space.interp_w(Code, w_code) if not space.is_true(space.isinstance(w_globals, space.w_dict)): @@ -229,7 +229,7 @@ return self.getrepr(self.space, 'function %s' % (self.name,)) - # delicate + # delicate _all = {'': None} def _freeze_(self): @@ -260,7 +260,7 @@ new_inst = mod.get('builtin_function') return space.newtuple([new_inst, space.newtuple([space.wrap(code.identifier)])]) - + new_inst = mod.get('func_new') w = space.wrap if self.closure is None: @@ -524,7 +524,7 @@ space = self.space other = space.interpclass_w(w_other) if not isinstance(other, Method): - return space.w_False + return space.w_NotImplemented if self.w_instance is None: if other.w_instance is not None: return space.w_False @@ -562,7 +562,7 @@ else: tup = [self.w_function, w_instance, self.w_class] return space.newtuple([new_inst, space.newtuple(tup)]) - + class StaticMethod(Wrappable): """The staticmethod objects.""" _immutable_ = True Modified: pypy/branch/fast-forward/pypy/interpreter/mixedmodule.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/mixedmodule.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/mixedmodule.py Sun Dec 12 18:11:19 2010 @@ -13,6 +13,10 @@ applevel_name = None expose__file__attribute = True + + # The following attribute is None as long as the module has not been + # imported yet, and when it has been, it is mod.__dict__.items() just + # after startup(). w_initialdict = None def __init__(self, space, w_name): @@ -26,8 +30,14 @@ """This is called each time the module is imported or reloaded """ if self.w_initialdict is not None: + # the module was already imported. Refresh its content with + # the saved dict, as done with built-in and extension modules + # on CPython. space.call_method(self.w_dict, 'update', self.w_initialdict) - Module.init(self, space) + else: + Module.init(self, space) + if not self.lazy and self.w_initialdict is None: + self.w_initialdict = space.call_method(self.w_dict, 'items') def get_applevel_name(cls): """ NOT_RPYTHON """ @@ -96,6 +106,7 @@ def _freeze_(self): self.getdict() + self.w_initialdict = None self.startup_called = False # hint for the annotator: Modules can hold state, so they are # not constant Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/error.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/pyparser/error.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/pyparser/error.py Sun Dec 12 18:11:19 2010 @@ -2,19 +2,22 @@ class SyntaxError(Exception): """Base class for exceptions raised by the parser.""" - def __init__(self, msg, lineno=0, offset=0, text=None, filename=None): + def __init__(self, msg, lineno=0, offset=0, text=None, filename=None, + lastlineno=0): self.msg = msg self.lineno = lineno self.offset = offset self.text = text self.filename = filename + self.lastlineno = lastlineno def wrap_info(self, space): return space.newtuple([space.wrap(self.msg), space.newtuple([space.wrap(self.filename), space.wrap(self.lineno), space.wrap(self.offset), - space.wrap(self.text)])]) + space.wrap(self.text), + space.wrap(self.lastlineno)])]) def __str__(self): return "%s at pos (%d, %d) in %r" % (self.__class__.__name__, @@ -33,8 +36,9 @@ class TokenError(SyntaxError): - def __init__(self, msg, line, lineno, column, tokens): - SyntaxError.__init__(self, msg, lineno, column, line) + def __init__(self, msg, line, lineno, column, tokens, lastlineno=0): + SyntaxError.__init__(self, msg, lineno, column, line, + lastlineno=lastlineno) self.tokens = tokens class TokenIndentationError(IndentationError): Modified: pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenizer.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenizer.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/pyparser/pytokenizer.py Sun Dec 12 18:11:19 2010 @@ -78,6 +78,7 @@ contline = None indents = [0] last_comment = '' + parenlevstart = (0, 0, "") # make the annotator happy endDFA = automata.DFA([], []) @@ -85,7 +86,7 @@ line = '' pos = 0 lines.append("") - strstart = (0, 0) + strstart = (0, 0, "") for line in lines: lnum = lnum + 1 pos, max = 0, len(line) @@ -94,7 +95,8 @@ if not line: raise TokenError( "EOF while scanning triple-quoted string literal", - line, lnum-1, 0, token_list) + strstart[2], strstart[0], strstart[1]+1, + token_list, lnum) endmatch = endDFA.recognize(line) if endmatch >= 0: pos = end = endmatch @@ -147,6 +149,10 @@ else: # continued statement if not line: + if parenlev > 0: + lnum1, start1, line1 = parenlevstart + raise TokenError("parenthesis is never closed", line1, + lnum1, start1 + 1, token_list, lnum) raise TokenError("EOF in multi-line statement", line, lnum, 0, token_list) continued = 0 @@ -188,7 +194,7 @@ token_list.append(tok) last_comment = '' else: - strstart = (lnum, start) + strstart = (lnum, start, line) contstr = line[start:] contline = line break @@ -196,7 +202,7 @@ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string - strstart = (lnum, start) + strstart = (lnum, start, line) endDFA = (endDFAs[initial] or endDFAs[token[1]] or endDFAs[token[2]]) contstr, needcont = line[start:], 1 @@ -213,6 +219,8 @@ continued = 1 else: if initial in '([{': + if parenlev == 0: + parenlevstart = (lnum, start, line) parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 @@ -231,7 +239,7 @@ start = pos if start', 'exec', 0) @@ -918,3 +929,25 @@ assert e.msg == 'unindent does not match any outer indentation level' else: raise Exception("DID NOT RAISE") + + + def test_repr_vs_str(self): + source1 = "x = (\n" + source2 = "x = (\n\n" + try: + exec source1 + except SyntaxError, err1: + pass + else: + raise Exception("DID NOT RAISE") + try: + exec source2 + except SyntaxError, err2: + pass + else: + raise Exception("DID NOT RAISE") + assert str(err1) != str(err2) + assert repr(err1) != repr(err2) + err3 = eval(repr(err1)) + assert str(err3) == str(err1) + assert repr(err3) == repr(err1) Modified: pypy/branch/fast-forward/pypy/interpreter/test/test_executioncontext.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/test/test_executioncontext.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/test/test_executioncontext.py Sun Dec 12 18:11:19 2010 @@ -7,6 +7,10 @@ class TestExecutionContext: + keywords = {} + + def setup_class(cls): + cls.space = gettestobjspace(**cls.keywords) def test_action(self): @@ -77,29 +81,43 @@ assert l == ['call', 'return', 'call', 'return'] def test_llprofile_c_call(self): + from pypy.interpreter.function import Function, Method l = [] + seen = [] + space = self.space - def profile_func(space, w_arg, frame, event, w_aarg): + def profile_func(space, w_arg, frame, event, w_func): assert w_arg is space.w_None l.append(event) + if event == 'c_call': + seen.append(w_func) - space = self.space - space.getexecutioncontext().setllprofile(profile_func, space.w_None) - - def check_snippet(snippet): + def check_snippet(snippet, expected_c_call): + del l[:] + del seen[:] + space.getexecutioncontext().setllprofile(profile_func, + space.w_None) space.appexec([], """(): %s return """ % snippet) space.getexecutioncontext().setllprofile(None, None) assert l == ['call', 'return', 'call', 'c_call', 'c_return', 'return'] - - check_snippet('l = []; l.append(42)') - check_snippet('max(1, 2)') - check_snippet('args = (1, 2); max(*args)') - check_snippet('max(1, 2, **{})') - check_snippet('args = (1, 2); max(*args, **{})') - check_snippet('abs(val=0)') + if isinstance(seen[0], Method): + found = 'method %s of %s' % ( + seen[0].w_function.name, + seen[0].w_class.getname(space, '?')) + else: + assert isinstance(seen[0], Function) + found = 'builtin %s' % seen[0].name + assert found == expected_c_call + + check_snippet('l = []; l.append(42)', 'method append of list') + check_snippet('max(1, 2)', 'builtin max') + check_snippet('args = (1, 2); max(*args)', 'builtin max') + check_snippet('max(1, 2, **{})', 'builtin max') + check_snippet('args = (1, 2); max(*args, **{})', 'builtin max') + check_snippet('abs(val=0)', 'builtin abs') def test_llprofile_c_exception(self): l = [] @@ -243,6 +261,13 @@ """) +class TestExecutionContextWithCallLikelyBuiltin(TestExecutionContext): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True} + +class TestExecutionContextWithCallMethod(TestExecutionContext): + keywords = {'objspace.opcodes.CALL_METHOD': True} + + class AppTestDelNotBlocked: def setup_method(self, meth): Modified: pypy/branch/fast-forward/pypy/interpreter/test/test_function.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/test/test_function.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/test/test_function.py Sun Dec 12 18:11:19 2010 @@ -6,7 +6,7 @@ from pypy.interpreter.argument import Arguments -class AppTestFunctionIntrospection: +class AppTestFunctionIntrospection: def test_attributes(self): globals()['__name__'] = 'mymodulename' def f(): pass @@ -90,10 +90,10 @@ def f(*args): return 42 raises(TypeError, "dir.func_code = f.func_code") - raises(TypeError, "list.append.im_func.func_code = f.func_code") + raises(TypeError, "list.append.im_func.func_code = f.func_code") -class AppTestFunction: +class AppTestFunction: def test_simple_call(self): def func(arg1, arg2): return arg1, arg2 @@ -118,7 +118,7 @@ assert res[2] == 333 raises(TypeError, func) - raises(TypeError, func, 1, 2, 3, 4) + raises(TypeError, func, 1, 2, 3, 4) def test_simple_varargs(self): def func(arg1, *args): @@ -129,7 +129,7 @@ res = func(23, *(42,)) assert res[0] == 23 - assert res[1] == (42,) + assert res[1] == (42,) def test_simple_kwargs(self): def func(arg1, **kwargs): @@ -222,7 +222,7 @@ func(**{'self': 23}) assert False except TypeError: - pass + pass def test_kwargs_confusing_name(self): def func(self): # 'self' conflicts with the interp-level @@ -304,7 +304,7 @@ # on function types raises(ValueError, type(f).__setstate__, f, (1, 2, 3)) -class AppTestMethod: +class AppTestMethod: def test_simple_call(self): class A(object): def func(self, arg2): @@ -325,7 +325,7 @@ res = a.func(*(42,)) assert res[0] is a - assert res[1] == (42,) + assert res[1] == (42,) def test_obscure_varargs(self): class A(object): @@ -338,14 +338,14 @@ res = a.func(*(42,)) assert res[0] is a - assert res[1] == 42 + assert res[1] == 42 def test_simple_kwargs(self): class A(object): def func(self, **kwargs): return self, kwargs a = A() - + res = a.func(value=42) assert res[0] is a assert res[1] == {'value': 42} @@ -399,19 +399,19 @@ assert hash(C.m) == hash(D.m) assert hash(c.m) == hash(c.m) - def test_method_repr(self): - class A(object): - def f(self): + def test_method_repr(self): + class A(object): + def f(self): pass assert repr(A.f) == "" - assert repr(A().f).startswith(">") + assert repr(A().f).startswith(">") class B: def f(self): pass assert repr(B.f) == "" assert repr(B().f).startswith(">") + assert repr(A().f).endswith(">>") def test_method_call(self): @@ -504,14 +504,33 @@ def f(): pass raises(TypeError, new.instancemethod, f, None) + def test_empty_arg_kwarg_call(self): + def f(): + pass + + raises(TypeError, lambda: f(*0)) + raises(TypeError, lambda: f(**0)) + + def test_method_equal(self): + class A(object): + def m(self): + pass + + class X(object): + def __eq__(self, other): + return True -class TestMethod: + assert A().m == X() + assert X() == A().m + + +class TestMethod: def setup_method(self, method): def c(self, bar): return bar code = PyCode._from_code(self.space, c.func_code) self.fn = Function(self.space, code, self.space.newdict()) - + def test_get(self): space = self.space w_meth = descr_function_get(space, self.fn, space.wrap(5), space.type(space.wrap(5))) @@ -569,7 +588,7 @@ def test_call_function(self): space = self.space - + d = {} for i in range(10): args = "(" + ''.join(["a%d," % a for a in range(i)]) + ")" @@ -591,14 +610,14 @@ code.funcrun = bomb code.funcrun_obj = bomb - args_w = map(space.wrap, range(i)) + args_w = map(space.wrap, range(i)) w_res = space.call_function(fn, *args_w) check = space.is_true(space.eq(w_res, space.wrap(res))) assert check def test_flatcall(self): space = self.space - + def f(a): return a code = PyCode._from_code(self.space, f.func_code) @@ -625,7 +644,7 @@ def test_flatcall_method(self): space = self.space - + def f(self, a): return a code = PyCode._from_code(self.space, f.func_code) @@ -653,7 +672,7 @@ def test_flatcall_default_arg(self): space = self.space - + def f(a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -682,7 +701,7 @@ def test_flatcall_default_arg_method(self): space = self.space - + def f(self, a, b): return a+b code = PyCode._from_code(self.space, f.func_code) @@ -705,7 +724,7 @@ y = A().m(x) b = A().m z = b(x) - return y+10*z + return y+10*z """) assert space.eq_w(w_res, space.wrap(44)) Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py Sun Dec 12 18:11:19 2010 @@ -17,7 +17,6 @@ from pypy.jit.backend.llsupport.descr import get_call_descr from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr -from pypy.jit.backend.llsupport.ffisupport import get_call_descr_dynamic from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager from pypy.rpython.annlowlevel import cast_instance_to_base_ptr @@ -247,7 +246,9 @@ return get_call_descr(self.gc_ll_descr, ARGS, RESULT, extrainfo) def calldescrof_dynamic(self, ffi_args, ffi_result, extrainfo=None): - return get_call_descr_dynamic(ffi_args, ffi_result, extrainfo) + from pypy.jit.backend.llsupport import ffisupport + return ffisupport.get_call_descr_dynamic(ffi_args, ffi_result, + extrainfo) def get_overflow_error(self): ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable) Modified: pypy/branch/fast-forward/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/compile.py Sun Dec 12 18:11:19 2010 @@ -73,6 +73,8 @@ op.setdescr(None) # clear reference, mostly for tests # mostly for tests: make sure we don't keep a reference to the LoopToken loop.token = None + if not we_are_translated(): + loop._number = looptoken.number # ____________________________________________________________ Modified: pypy/branch/fast-forward/pypy/jit/metainterp/graphpage.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/graphpage.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/graphpage.py Sun Dec 12 18:11:19 2010 @@ -31,8 +31,10 @@ def compute(self, graphs, errmsg=None): resopgen = ResOpGen() for graph, highlight in graphs: - if hasattr(graph, 'token'): + if getattr(graph, 'token', None) is not None: resopgen.jumps_to_graphs[graph.token] = graph + if getattr(graph, '_number', None) is not None: + resopgen.jumps_to_graphs[graph._number] = graph for graph, highlight in graphs: resopgen.add_graph(graph, highlight) @@ -173,7 +175,10 @@ if tgt is None: tgt_g = graphindex else: - tgt = self.jumps_to_graphs.get(tgt) + if tgt in self.jumps_to_graphs: + tgt = self.jumps_to_graphs[tgt] + else: + tgt = self.jumps_to_graphs.get(tgt.number) if tgt is not None: tgt_g = self.graphs.index(tgt) if tgt_g != -1: Modified: pypy/branch/fast-forward/pypy/jit/metainterp/optimizeopt/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/optimizeopt/__init__.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/optimizeopt/__init__.py Sun Dec 12 18:11:19 2010 @@ -3,7 +3,6 @@ from pypy.jit.metainterp.optimizeopt.intbounds import OptIntBounds from pypy.jit.metainterp.optimizeopt.virtualize import OptVirtualize from pypy.jit.metainterp.optimizeopt.heap import OptHeap -from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall from pypy.jit.metainterp.optimizeopt.string import OptString def optimize_loop_1(metainterp_sd, loop, virtuals=True): @@ -17,6 +16,10 @@ OptVirtualize(), OptString(), OptHeap(), + ] + if metainterp_sd.jit_ffi: + from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall + optimizations = optimizations + [ OptFfiCall(), ] optimizer = Optimizer(metainterp_sd, loop, optimizations, virtuals) Modified: pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py Sun Dec 12 18:11:19 2010 @@ -1216,7 +1216,8 @@ logger_ops = None def __init__(self, cpu, options, - ProfilerClass=EmptyProfiler, warmrunnerdesc=None): + ProfilerClass=EmptyProfiler, warmrunnerdesc=None, + jit_ffi=True): self.cpu = cpu self.stats = self.cpu.stats self.options = options @@ -1226,6 +1227,7 @@ self.profiler = ProfilerClass() self.profiler.cpu = cpu self.warmrunnerdesc = warmrunnerdesc + self.jit_ffi = jit_ffi backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_compile.py Sun Dec 12 18:11:19 2010 @@ -54,6 +54,7 @@ stats = Stats() profiler = jitprof.EmptyProfiler() warmrunnerdesc = None + jit_ffi = False def log(self, msg, event_kind=None): pass Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_fficall.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_fficall.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_fficall.py Sun Dec 12 18:11:19 2010 @@ -40,6 +40,6 @@ n += 1 return res # - res = self.meta_interp(f, [0]) + res = self.meta_interp(f, [0], jit_ffi=True) return res Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizefficall.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizefficall.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizefficall.py Sun Dec 12 18:11:19 2010 @@ -31,6 +31,7 @@ class TestFfiCall(BaseTestOptimizeOpt, LLtypeMixin): + jit_ffi = True class namespace: cpu = LLtypeMixin.cpu Modified: pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizeopt.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizeopt.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/test/test_optimizeopt.py Sun Dec 12 18:11:19 2010 @@ -28,11 +28,12 @@ class FakeMetaInterpStaticData(object): - def __init__(self, cpu): + def __init__(self, cpu, jit_ffi=False): self.cpu = cpu self.profiler = EmptyProfiler() self.options = Fake() self.globaldata = Fake() + self.jit_ffi = jit_ffi def test_store_final_boxes_in_guard(): from pypy.jit.metainterp.compile import ResumeGuardDescr @@ -227,6 +228,7 @@ return sorted(boxes, key=lambda box: _kind2count[box.type]) class BaseTestOptimizeOpt(BaseTest): + jit_ffi = False def invent_fail_descr(self, fail_args): if fail_args is None: @@ -261,7 +263,7 @@ loop.token.specnodes = self.unpack_specnodes(spectext) # self.loop = loop - metainterp_sd = FakeMetaInterpStaticData(self.cpu) + metainterp_sd = FakeMetaInterpStaticData(self.cpu, self.jit_ffi) if hasattr(self, 'vrefinfo'): metainterp_sd.virtualref_info = self.vrefinfo if hasattr(self, 'callinfocollection'): Modified: pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/warmspot.py Sun Dec 12 18:11:19 2010 @@ -144,7 +144,8 @@ class WarmRunnerDesc(object): def __init__(self, translator, policy=None, backendopt=True, CPUClass=None, - optimizer=None, ProfilerClass=EmptyProfiler, **kwds): + optimizer=None, ProfilerClass=EmptyProfiler, + jit_ffi=None, **kwds): pyjitpl._warmrunnerdesc = self # this is a global for debugging only! self.set_translator(translator) self.memory_manager = memmgr.MemoryManager() @@ -162,7 +163,7 @@ elif self.opt.listops: self.prejit_optimizations_minimal_inline(policy, graphs) - self.build_meta_interp(ProfilerClass) + self.build_meta_interp(ProfilerClass, jit_ffi) self.make_args_specifications() # from pypy.jit.metainterp.virtualref import VirtualRefInfo @@ -280,11 +281,14 @@ translate_support_code, gcdescr=self.gcdescr) self.cpu = cpu - def build_meta_interp(self, ProfilerClass): + def build_meta_interp(self, ProfilerClass, jit_ffi=None): + if jit_ffi is None: + jit_ffi = self.translator.config.translation.jit_ffi self.metainterp_sd = MetaInterpStaticData(self.cpu, self.opt, ProfilerClass=ProfilerClass, - warmrunnerdesc=self) + warmrunnerdesc=self, + jit_ffi=jit_ffi) def make_virtualizable_infos(self): vinfos = {} Modified: pypy/branch/fast-forward/pypy/module/_lsprof/interp_lsprof.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_lsprof/interp_lsprof.py (original) +++ pypy/branch/fast-forward/pypy/module/_lsprof/interp_lsprof.py Sun Dec 12 18:11:19 2010 @@ -164,8 +164,11 @@ if isinstance(w_arg, Method): w_function = w_arg.w_function class_name = w_arg.w_class.getname(space, '?') - assert isinstance(w_function, Function) - return "{method '%s' of '%s' objects}" % (w_function.name, class_name) + if isinstance(w_function, Function): + name = w_function.name + else: + name = '?' + return "{method '%s' of '%s' objects}" % (name, class_name) elif isinstance(w_arg, Function): if w_arg.w_module is None: module = '' @@ -177,7 +180,8 @@ module += '.' return '{%s%s}' % (module, w_arg.name) else: - return '{!!!unknown!!!}' + class_name = space.type(w_arg).getname(space, '?') + return "{'%s' object}" % (class_name,) def lsprof_call(space, w_self, frame, event, w_arg): assert isinstance(w_self, W_Profiler) Modified: pypy/branch/fast-forward/pypy/module/_lsprof/test/test_cprofile.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_lsprof/test/test_cprofile.py (original) +++ pypy/branch/fast-forward/pypy/module/_lsprof/test/test_cprofile.py Sun Dec 12 18:11:19 2010 @@ -2,9 +2,10 @@ from pypy.conftest import gettestobjspace, option class AppTestCProfile(object): + keywords = {} def setup_class(cls): - space = gettestobjspace(usemodules=('_lsprof',)) + space = gettestobjspace(usemodules=('_lsprof',), **cls.keywords) cls.w_expected_output = space.wrap(expected_output) cls.space = space cls.w_file = space.wrap(__file__) @@ -148,6 +149,12 @@ finally: sys.path.pop(0) + +class AppTestWithDifferentBytecodes(AppTestCProfile): + keywords = {'objspace.opcodes.CALL_LIKELY_BUILTIN': True, + 'objspace.opcodes.CALL_METHOD': True} + + expected_output = {} expected_output['print_stats'] = """\ 126 function calls (106 primitive calls) in 1.000 CPU seconds @@ -165,11 +172,11 @@ 2 0.000 0.000 0.140 0.070 profilee.py:84(helper2_indirect) 8 0.312 0.039 0.400 0.050 profilee.py:88(helper2) 8 0.064 0.008 0.080 0.010 profilee.py:98(subhelper) - 4 0.000 0.000 0.000 0.000 {.*append.*} + 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} 1 0.000 0.000 0.000 0.000 {.*disable.*} - 12 0.000 0.000 0.012 0.001 {hasattr.*} - 8 0.000 0.000 0.000 0.000 {range.*} - 4 0.000 0.000 0.000 0.000 {sys.exc_info.*} + 12 0.000 0.000 0.012 0.001 {hasattr} + 8 0.000 0.000 0.000 0.000 {range} + 4 0.000 0.000 0.000 0.000 {sys.exc_info} """ Modified: pypy/branch/fast-forward/pypy/module/cpyext/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/__init__.py Sun Dec 12 18:11:19 2010 @@ -47,6 +47,7 @@ import pypy.module.cpyext.weakrefobject import pypy.module.cpyext.funcobject import pypy.module.cpyext.classobject +import pypy.module.cpyext.pypyintf import pypy.module.cpyext.memoryobject import pypy.module.cpyext.codecs Modified: pypy/branch/fast-forward/pypy/module/cpyext/src/getargs.c ============================================================================== --- pypy/branch/fast-forward/pypy/module/cpyext/src/getargs.c (original) +++ pypy/branch/fast-forward/pypy/module/cpyext/src/getargs.c Sun Dec 12 18:11:19 2010 @@ -459,6 +459,7 @@ bufsize); return msgbuf; } + PyPy_Borrow(arg, item); msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); if (msg != NULL) { Modified: pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py (original) +++ pypy/branch/fast-forward/pypy/module/exceptions/interp_exceptions.py Sun Dec 12 18:11:19 2010 @@ -505,6 +505,7 @@ self.w_text = space.w_None self.w_msg = space.w_None self.w_print_file_and_line = space.w_None # what's that? + self.w_lastlineno = space.w_None # this is a pypy extension W_BaseException.__init__(self, space) def descr_init(self, space, args_w): @@ -512,11 +513,16 @@ if len(args_w) > 0: self.w_msg = args_w[0] if len(args_w) == 2: - values_w = space.fixedview(args_w[1], 4) - self.w_filename = values_w[0] - self.w_lineno = values_w[1] - self.w_offset = values_w[2] - self.w_text = values_w[3] + values_w = space.fixedview(args_w[1]) + if len(values_w) > 0: self.w_filename = values_w[0] + if len(values_w) > 1: self.w_lineno = values_w[1] + if len(values_w) > 2: self.w_offset = values_w[2] + if len(values_w) > 3: self.w_text = values_w[3] + if len(values_w) > 4: + self.w_lastlineno = values_w[4] # PyPy extension + # kill the extra items from args_w to prevent undesired effects + args_w = args_w[:] + args_w[1] = space.newtuple(values_w[:4]) W_BaseException.descr_init(self, space, args_w) descr_init.unwrap_spec = ['self', ObjSpace, 'args_w'] @@ -525,29 +531,52 @@ if type(self.msg) is not str: return str(self.msg) + lineno = None buffer = self.msg have_filename = type(self.filename) is str - have_lineno = type(self.lineno) is int + if type(self.lineno) is int: + if (type(self.lastlineno) is int and + self.lastlineno > self.lineno): + lineno = 'lines %d-%d' % (self.lineno, self.lastlineno) + else: + lineno = 'line %d' % (self.lineno,) if have_filename: import os fname = os.path.basename(self.filename or "???") - if have_lineno: - buffer = "%s (%s, line %ld)" % (self.msg, fname, self.lineno) + if lineno: + buffer = "%s (%s, %s)" % (self.msg, fname, lineno) else: buffer ="%s (%s)" % (self.msg, fname) - elif have_lineno: - buffer = "%s (line %ld)" % (self.msg, self.lineno) + elif lineno: + buffer = "%s (%s)" % (self.msg, lineno) return buffer """) descr_str.unwrap_spec = ['self', ObjSpace] + def descr_repr(self, space): + if (len(self.args_w) == 2 + and not space.is_w(self.w_lastlineno, space.w_None) + and space.int_w(space.len(self.args_w[1])) == 4): + # fake a 5-element tuple in the repr, suitable for calling + # __init__ again + values_w = space.fixedview(self.args_w[1]) + w_tuple = space.newtuple(values_w + [self.w_lastlineno]) + args_w = [self.args_w[0], w_tuple] + args_repr = space.str_w(space.repr(space.newtuple(args_w))) + clsname = self.getclass(space).getname(space, '?') + return space.wrap(clsname + args_repr) + else: + return W_StandardError.descr_repr(self, space) + descr_repr.unwrap_spec = ['self', ObjSpace] + W_SyntaxError.typedef = TypeDef( 'SyntaxError', W_StandardError.typedef, __new__ = _new(W_SyntaxError), __init__ = interp2app(W_SyntaxError.descr_init), __str__ = interp2app(W_SyntaxError.descr_str), + __repr__ = interp2app(W_SyntaxError.descr_repr), __doc__ = W_SyntaxError.__doc__, __module__ = 'exceptions', msg = readwrite_attrproperty_w('w_msg', W_SyntaxError), @@ -557,6 +586,7 @@ text = readwrite_attrproperty_w('w_text', W_SyntaxError), print_file_and_line = readwrite_attrproperty_w('w_print_file_and_line', W_SyntaxError), + lastlineno = readwrite_attrproperty_w('w_lastlineno', W_SyntaxError), ) W_FutureWarning = _new_exception('FutureWarning', W_Warning, Modified: pypy/branch/fast-forward/pypy/module/imp/importing.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/imp/importing.py (original) +++ pypy/branch/fast-forward/pypy/module/imp/importing.py Sun Dec 12 18:11:19 2010 @@ -564,7 +564,7 @@ space.sys.setmodule(w_module) raise finally: - space.reloading_modules.clear() + del space.reloading_modules[modulename] # __________________________________________________________________ Modified: pypy/branch/fast-forward/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/__init__.py Sun Dec 12 18:11:19 2010 @@ -77,6 +77,8 @@ interpleveldefs['fsync'] = 'interp_posix.fsync' if hasattr(os, 'fdatasync'): interpleveldefs['fdatasync'] = 'interp_posix.fdatasync' + if hasattr(os, 'fchdir'): + interpleveldefs['fchdir'] = 'interp_posix.fchdir' if hasattr(os, 'putenv'): interpleveldefs['putenv'] = 'interp_posix.putenv' if hasattr(posix, 'unsetenv'): # note: emulated in os @@ -113,6 +115,8 @@ interpleveldefs['sysconf_names'] = 'space.wrap(os.sysconf_names)' if hasattr(os, 'ttyname'): interpleveldefs['ttyname'] = 'interp_posix.ttyname' + if hasattr(os, 'getloadavg'): + interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', Modified: pypy/branch/fast-forward/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/interp_posix.py Sun Dec 12 18:11:19 2010 @@ -4,6 +4,7 @@ from pypy.rlib.rarithmetic import r_longlong from pypy.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 +from pypy.interpreter.error import operationerrfmt from pypy.rpython.module.ll_os import RegisterOs from pypy.rpython.module import ll_os_stat from pypy.rpython.lltypesystem import rffi, lltype @@ -159,19 +160,29 @@ raise wrap_oserror(space, e) ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] -def fsync(space, fd): +def fsync(space, w_fd): + fd = space.c_filedescriptor_w(w_fd) try: os.fsync(fd) except OSError, e: raise wrap_oserror(space, e) -fsync.unwrap_spec = [ObjSpace, "c_int"] +fsync.unwrap_spec = [ObjSpace, W_Root] -def fdatasync(space, fd): +def fdatasync(space, w_fd): + fd = space.c_filedescriptor_w(w_fd) try: os.fdatasync(fd) except OSError, e: raise wrap_oserror(space, e) -fdatasync.unwrap_spec = [ObjSpace, "c_int"] +fdatasync.unwrap_spec = [ObjSpace, W_Root] + +def fchdir(space, w_fd): + fd = space.c_filedescriptor_w(w_fd) + try: + os.fchdir(fd) + except OSError, e: + raise wrap_oserror(space, e) +fchdir.unwrap_spec = [ObjSpace, W_Root] # ____________________________________________________________ @@ -1023,6 +1034,17 @@ return space.w_None chown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] +def getloadavg(space): + try: + load = os.getloadavg() + except OSError, e: + raise OperationError(space.w_OSError, + space.wrap("Load averages are unobtainable")) + return space.newtuple([space.wrap(load[0]), + space.wrap(load[1]), + space.wrap(load[2])]) +getloadavg.unwrap_spec = [ObjSpace] + if _WIN: from pypy.rlib import rwin32 Modified: pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py Sun Dec 12 18:11:19 2010 @@ -534,6 +534,14 @@ assert os.WIFEXITED(status) assert os.WEXITSTATUS(status) == exit_status + if hasattr(os, 'getloadavg'): + def test_os_getloadavg(self): + os = self.posix + l0, l1, l2 = os.getloadavg() + assert type(l0) is float and l0 >= 0.0 + assert type(l1) is float and l0 >= 0.0 + assert type(l2) is float and l0 >= 0.0 + if hasattr(os, 'fsync'): def test_fsync(self): os = self.posix @@ -541,30 +549,42 @@ try: fd = f.fileno() os.fsync(fd) - finally: + os.fsync(long(fd)) + os.fsync(f) # <- should also work with a file, or anything + finally: # with a fileno() method f.close() - try: - os.fsync(fd) - except OSError: - pass - else: - raise AssertionError("os.fsync didn't raise") + raises(OSError, os.fsync, fd) + raises(ValueError, os.fsync, -1) if hasattr(os, 'fdatasync'): def test_fdatasync(self): os = self.posix - f = open(self.path2) + f = open(self.path2, "w") try: fd = f.fileno() os.fdatasync(fd) finally: f.close() + raises(OSError, os.fdatasync, fd) + raises(ValueError, os.fdatasync, -1) + + if hasattr(os, 'fchdir'): + def test_fchdir(self): + os = self.posix + localdir = os.getcwd() try: - os.fdatasync(fd) - except OSError: - pass - else: - raise AssertionError("os.fdatasync didn't raise") + os.mkdir(self.path2 + 'dir') + fd = os.open(self.path2 + 'dir', os.O_RDONLY) + try: + os.fchdir(fd) + mypath = os.getcwd() + finally: + os.close(fd) + assert mypath.endswith('test_posix2-dir') + raises(OSError, os.fchdir, fd) + raises(ValueError, os.fchdir, -1) + finally: + os.chdir(localdir) def test_largefile(self): os = self.posix Modified: pypy/branch/fast-forward/pypy/module/select/interp_select.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/select/interp_select.py (original) +++ pypy/branch/fast-forward/pypy/module/select/interp_select.py Sun Dec 12 18:11:19 2010 @@ -14,32 +14,12 @@ unregistering file descriptors, and then polling them for I/O events.""" return Poll() -def as_fd_w(space, w_fd): - if not space.is_true(space.isinstance(w_fd, space.w_int)): - try: - w_fileno = space.getattr(w_fd, space.wrap('fileno')) - except OperationError, e: - if e.match(space, space.w_AttributeError): - raise OperationError(space.w_TypeError, - space.wrap("argument must be an int, or have a fileno() method.")) - raise - w_fd = space.call_function(w_fileno) - if not space.is_true(space.isinstance(w_fd, space.w_int)): - raise OperationError(space.w_TypeError, - space.wrap('filneo() return a non-integer')) - - fd = space.int_w(w_fd) - if fd < 0: - raise operationerrfmt(space.w_ValueError, - "file descriptor cannot be a negative integer (%d)", fd) - return fd - class Poll(Wrappable): def __init__(self): self.fddict = {} def register(self, space, w_fd, events=defaultevents): - fd = as_fd_w(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) self.fddict[fd] = events register.unwrap_spec = ['self', ObjSpace, W_Root, int] @@ -52,7 +32,7 @@ modify.unwrap_spec = ['self', ObjSpace, W_Root, int] def unregister(self, space, w_fd): - fd = as_fd_w(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) try: del self.fddict[fd] except KeyError: @@ -123,9 +103,9 @@ iwtd_w = space.listview(w_iwtd) owtd_w = space.listview(w_owtd) ewtd_w = space.listview(w_ewtd) - iwtd = [as_fd_w(space, w_f) for w_f in iwtd_w] - owtd = [as_fd_w(space, w_f) for w_f in owtd_w] - ewtd = [as_fd_w(space, w_f) for w_f in ewtd_w] + iwtd = [space.c_filedescriptor_w(w_f) for w_f in iwtd_w] + owtd = [space.c_filedescriptor_w(w_f) for w_f in owtd_w] + ewtd = [space.c_filedescriptor_w(w_f) for w_f in ewtd_w] iwtd_d = {} owtd_d = {} ewtd_d = {} Modified: pypy/branch/fast-forward/pypy/module/sys/vm.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/vm.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/vm.py Sun Dec 12 18:11:19 2010 @@ -51,7 +51,7 @@ space.wrap("recursion limit must be positive")) # global recursion_limit # we need to do it without writing globals. - space.warn('setrecursionlimit deprecated', space.w_DeprecationWarning) + space.warn('setrecursionlimit() is ignored (and not needed) on PyPy', space.w_DeprecationWarning) space.sys.recursionlimit = new_limit def getrecursionlimit(space): Modified: pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py (original) +++ pypy/branch/fast-forward/pypy/module/test_lib_pypy/ctypes_tests/test_cast.py Sun Dec 12 18:11:19 2010 @@ -82,8 +82,11 @@ def test_cast_functype(self): # make sure we can cast function type my_sqrt = lib.my_sqrt + saved_objects = my_sqrt._objects.copy() sqrt = cast(cast(my_sqrt, c_void_p), CFUNCTYPE(c_double, c_double)) assert sqrt(4.0) == 2.0 assert not cast(0, CFUNCTYPE(c_int)) - - + # + assert sqrt._objects is my_sqrt._objects # on CPython too + my_sqrt._objects.clear() + my_sqrt._objects.update(saved_objects) Modified: pypy/branch/fast-forward/pypy/objspace/std/complexobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/complexobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/complexobject.py Sun Dec 12 18:11:19 2010 @@ -25,90 +25,92 @@ """ representation for debugging purposes """ return "" % (w_self.realval, w_self.imagval) -registerimplementation(W_ComplexObject) - -c_1 = (1.0, 0.0) + def sub(self, other): + return W_ComplexObject(self.realval - other.realval, + self.imagval - other.imagval) + + def mul(self, other): + r = self.realval * other.realval - self.imagval * other.imagval + i = self.realval * other.imagval + self.imagval * other.realval + return W_ComplexObject(r, i) + + def div(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 < 0: + abs_r2 = - r2 + else: + abs_r2 = r2 + if i2 < 0: + abs_i2 = - i2 + else: + abs_i2 = i2 + if abs_r2 >= abs_i2: + if abs_r2 == 0.0: + raise ZeroDivisionError + else: + ratio = i2 / r2 + denom = r2 + i2 * ratio + rr = (r1 + i1 * ratio) / denom + ir = (i1 - r1 * ratio) / denom + else: + ratio = r2 / i2 + denom = r2 * ratio + i2 + assert i2 != 0.0 + rr = (r1 * ratio + i1) / denom + ir = (i1 * ratio - r1) / denom + return W_ComplexObject(rr,ir) + + def divmod(self, other): + w_div = self.div(other) + div = math.floor(w_div.realval) + w_mod = self.sub( + W_ComplexObject(other.realval * div, other.imagval * div)) + return (W_ComplexObject(div, 0), w_mod) + + def pow(self, other): + r1, i1 = self.realval, self.imagval + r2, i2 = other.realval, other.imagval + if r2 == 0.0 and i2 == 0.0: + rr, ir = 1, 0 + elif r1 == 0.0 and i1 == 0.0: + if i2 != 0.0 or r2 < 0.0: + raise ZeroDivisionError + rr, ir = (0.0, 0.0) + else: + vabs = math.hypot(r1,i1) + len = math.pow(vabs,r2) + at = math.atan2(i1,r1) + phase = at * r2 + if i2 != 0.0: + len /= math.exp(at * i2) + phase += i2 * math.log(vabs) + rr = len * math.cos(phase) + ir = len * math.sin(phase) + return W_ComplexObject(rr, ir) + + def pow_int(self, n): + if n > 100 or n < -100: + return self.pow(W_ComplexObject(1.0 * n, 0.0)) + elif n > 0: + return self.pow_positive_int(n) + else: + return w_one.div(self.pow_positive_int(-n)) -def _sum(c1, c2): - return (c1[0]+c2[0],c1[1]+c2[1]) + def pow_positive_int(self, n): + mask = 1 + w_result = w_one + while mask > 0 and n >= mask: + if n & mask: + w_result = w_result.mul(self) + mask <<= 1 + self = self.mul(self) -def _diff(c1, c2): - return (c1[0]-c2[0],c1[1]-c2[1]) + return w_result -def _prod(c1, c2): - r = c1[0]*c2[0] - c1[1]*c2[1] - i = c1[0]*c2[1] + c1[1]*c2[0] - return (r,i) - -def _quot(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 < 0: - abs_r2 = - r2 - else: - abs_r2 = r2 - if i2 < 0: - abs_i2 = - i2 - else: - abs_i2 = i2 - if abs_r2 >= abs_i2: - if abs_r2 == 0.0: - raise ZeroDivisionError - else: - ratio = i2 / r2 - denom = r2 + i2 * ratio - rr = (r1 + i1 * ratio) / denom - ir = (i1 - r1 * ratio) / denom - else: - ratio = r2 / i2 - denom = r2 * ratio + i2 - assert i2 != 0.0 - rr = (r1 * ratio + i1) / denom - ir = (i1 * ratio - r1) / denom - return (rr,ir) - -def _pow(c1,c2): - r1, i1 = c1 - r2, i2 = c2 - if r2 == 0.0 and i2 == 0.0: - rr, ir = c_1 - elif r1 == 0.0 and i1 == 0.0: - if i2 != 0.0 or r2 < 0.0: - raise ZeroDivisionError - rr, ir = (0.0, 0.0) - else: - vabs = math.hypot(r1,i1) - len = math.pow(vabs,r2) - at = math.atan2(i1,r1) - phase = at * r2 - if i2 != 0.0: - len /= math.exp(at * i2) - phase += i2 * math.log(vabs) - rr = len * math.cos(phase) - ir = len * math.sin(phase) - return (rr, ir) - -def _powu(c,n): - mask = 1; - rr, ir = c_1 - rp = c[0] - ip = c[1] - while mask > 0 and n >= mask: - if n & mask: - rr, ir = _prod((rr, ir), (rp, ip)) - mask <<= 1 - rp, ip = _prod((rp, ip), (rp, ip)) - - return (rr, ir) - -def _powi(c,n): - if n > 100 or n < -100: - return _pow(c,(1.0 * n, 0.0)) - elif n > 0: - return _powu(c, n) - else: - return _quot(c_1, _powu(c, -n)) +registerimplementation(W_ComplexObject) +w_one = W_ComplexObject(1, 0) def delegate_Bool2Complex(space, w_bool): @@ -128,38 +130,25 @@ return W_ComplexObject(w_float.floatval, 0.0) def hash__Complex(space, w_value): - #this is straight out of CPython complex implementation - hashreal = _hash_float(space, w_value.realval) - if hashreal == -1: - return space.newint(-1) hashimg = _hash_float(space, w_value.imagval) - if hashimg == -1: - return space.newint(-1) combined = hashreal + 1000003 * hashimg - if (combined == -1): - combined = -2 return space.newint(combined) -def _w2t(space, w_complex): - "convert an interplevel complex object to a tuple representation" - return w_complex.realval, w_complex.imagval - -def _t2w(space, c): - return W_ComplexObject(c[0], c[1]) - def add__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _sum(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval + w_complex2.realval, + w_complex1.imagval + w_complex2.imagval) def sub__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _diff(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return W_ComplexObject(w_complex1.realval - w_complex2.realval, + w_complex1.imagval - w_complex2.imagval) def mul__Complex_Complex(space, w_complex1, w_complex2): - return _t2w(space, _prod(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.mul(w_complex2) def div__Complex_Complex(space, w_complex1, w_complex2): try: - return _t2w(space, _quot(_w2t(space, w_complex1), _w2t(space, w_complex2))) + return w_complex1.div(w_complex2) except ZeroDivisionError, e: raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) @@ -167,49 +156,38 @@ def mod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[1] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex remainder")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - - return _t2w(space, mod) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) def divmod__Complex_Complex(space, w_complex1, w_complex2): try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + div, mod = w_complex1.divmod(w_complex2) except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex divmod()")) - div = (math.floor(div[0]), 0.0) - mod = _diff(_w2t(space, w_complex1), _prod(_w2t(space, w_complex2), div)) - w_div = _t2w(space, div) - w_mod = _t2w(space, mod) - return space.newtuple([w_div, w_mod]) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) + return space.newtuple([div, mod]) def floordiv__Complex_Complex(space, w_complex1, w_complex2): + # don't care about the slight slowdown you get from using divmod try: - div = _quot(_w2t(space, w_complex1), _w2t(space, w_complex2)) + return w_complex1.divmod(w_complex2)[0] except ZeroDivisionError, e: - raise OperationError(space.w_ZeroDivisionError, space.wrap("complex floordiv()")) - div = (math.floor(div[0]), 0.0) - return _t2w(space, div) + raise OperationError(space.w_ZeroDivisionError, space.wrap(str(e))) -def pow__Complex_Complex_ANY(space, w_complex1, w_complex2, thirdArg): +def pow__Complex_Complex_ANY(space, w_complex, w_exponent, thirdArg): if not space.is_w(thirdArg, space.w_None): raise OperationError(space.w_ValueError, space.wrap('complex modulo')) + int_exponent = int(w_exponent.realval) try: - v = _w2t(space, w_complex1) - exponent = _w2t(space, w_complex2) - int_exponent = int(exponent[0]) - if exponent[1] == 0.0 and exponent[0] == int_exponent: - p = _powi(v, int_exponent) + if w_exponent.imagval == 0.0 and w_exponent.realval == int_exponent: + w_p = w_complex.pow_int(int_exponent) else: - p = _pow(v, exponent) + w_p = w_complex.pow(w_exponent) except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, space.wrap("0.0 to a negative or complex power")) except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("complex exponentiation")) - return _t2w(space, p) + return w_p def neg__Complex(space, w_complex): return W_ComplexObject(-w_complex.realval, -w_complex.imagval) Modified: pypy/branch/fast-forward/pypy/objspace/std/complextype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/complextype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/complextype.py Sun Dec 12 18:11:19 2010 @@ -1,7 +1,7 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.objspace.std.register_all import register_all -from pypy.objspace.std.strutil import interp_string_to_float, ParseStringError +from pypy.objspace.std.strutil import string_to_float, ParseStringError from pypy.objspace.std.noneobject import W_NoneObject from pypy.objspace.std.stdtypedef import GetSetProperty, StdTypeDef from pypy.objspace.std.stdtypedef import StdObjSpaceMultiMethod @@ -135,8 +135,8 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) try: - realval = interp_string_to_float(space, realstr) - imagval = interp_string_to_float(space, imagstr) + realval = string_to_float(realstr) + imagval = string_to_float(imagstr) except ParseStringError: raise OperationError(space.w_ValueError, space.wrap(ERR_MALFORMED)) else: Modified: pypy/branch/fast-forward/pypy/objspace/std/floattype.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/floattype.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/floattype.py Sun Dec 12 18:11:19 2010 @@ -8,7 +8,7 @@ from pypy.objspace.std.register_all import register_all from pypy.objspace.std.stdtypedef import StdTypeDef, SMM from pypy.objspace.std.strutil import ParseStringError -from pypy.objspace.std.strutil import interp_string_to_float +from pypy.objspace.std.strutil import string_to_float float_as_integer_ratio = SMM("as_integer_ratio", 1) @@ -37,7 +37,7 @@ elif space.is_true(space.isinstance(w_value, space.w_str)): strvalue = space.str_w(w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) @@ -48,7 +48,7 @@ from unicodeobject import unicode_to_decimal_w strvalue = unicode_to_decimal_w(space, w_value) try: - value = interp_string_to_float(space, strvalue) + value = string_to_float(strvalue) except ParseStringError, e: raise OperationError(space.w_ValueError, space.wrap(e.msg)) Modified: pypy/branch/fast-forward/pypy/objspace/std/longobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/longobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/longobject.py Sun Dec 12 18:11:19 2010 @@ -45,19 +45,6 @@ fromrarith_int._annspecialcase_ = "specialize:argtype(0)" fromrarith_int = staticmethod(fromrarith_int) - def fromdecimalstr(s): - return W_LongObject(rbigint.fromdecimalstr(s)) - fromdecimalstr = staticmethod(fromdecimalstr) - - def _count_bits(self): - return self.num._count_bits() - - def is_odd(self): - return self.num.is_odd() - - def get_sign(self): - return self.num.sign - registerimplementation(W_LongObject) # bool-to-long Modified: pypy/branch/fast-forward/pypy/objspace/std/strutil.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/strutil.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/strutil.py Sun Dec 12 18:11:19 2010 @@ -156,7 +156,7 @@ del calc_mantissa_bits MANTISSA_DIGITS = len(str( (1L << MANTISSA_BITS)-1 )) + 1 -def interp_string_to_float(space, s): +def string_to_float(s): """ Conversion of string to float. This version tries to only raise on invalid literals. @@ -168,10 +168,9 @@ s = strip_spaces(s) if not s: - raise OperationError(space.w_ValueError, space.wrap( - "empty string for float()")) + raise ParseStringError("empty string for float()") + - low = s.lower() if low == "-inf" or low == "-infinity": return -INFINITY @@ -212,68 +211,56 @@ if len(digits) == 0: digits = '0' - # a few abbreviations - from pypy.objspace.std import longobject - mklong = longobject.W_LongObject.fromint - d2long = longobject.W_LongObject.fromdecimalstr - adlong = longobject.add__Long_Long - longup = longobject.pow__Long_Long_None - multip = longobject.mul__Long_Long - divide = longobject.div__Long_Long - lshift = longobject.lshift__Long_Long - rshift = longobject.rshift__Long_Long - # 4) compute the exponent and truncate to +-400 if not exponent: exponent = '0' - w_le = d2long(exponent) - w_le = adlong(space, w_le, mklong(space, dexp)) + long_exponent = rbigint.fromdecimalstr(exponent) + long_exponent = long_exponent.add(rbigint.fromint(dexp)) try: - e = w_le.toint() + e = long_exponent.toint() except OverflowError: # XXX poking at internals - e = w_le.num.sign * 400 - if e >= 400: - e = 400 - elif e <= -400: - e = -400 + e = long_exponent.sign * 400 + else: + if e >= 400: + e = 400 + elif e <= -400: + e = -400 # 5) compute the value using long math and proper rounding. - w_lr = d2long(digits) - w_10 = mklong(space, 10) - w_1 = mklong(space, 1) + b_digits = rbigint.fromdecimalstr(digits) + b_10 = rbigint.fromint(10) + b_1 = rbigint.fromint(1) if e >= 0: bits = 0 - w_pten = longup(space, w_10, mklong(space, e), space.w_None) - w_m = multip(space, w_lr, w_pten) + b_power_of_ten = b_10.pow(rbigint.fromint(e)) + b_mantissa = b_digits.mul(b_power_of_ten) else: # compute a sufficiently large scale prec = MANTISSA_DIGITS * 2 + 22 # 128, maybe bits = - (int(math.ceil(-e / math.log10(2.0) - 1e-10)) + prec) - w_scale = lshift(space, w_1, mklong(space, -bits)) - w_pten = longup(space, w_10, mklong(space, -e), None) - w_tmp = multip(space, w_lr, w_scale) - w_m = divide(space, w_tmp, w_pten) + b_scale = b_1.lshift(-bits) + b_power_of_ten = b_10.pow(rbigint.fromint(-e)) + b_mantissa = b_digits.mul(b_scale).div(b_power_of_ten) # we now have a fairly large mantissa. # Shift it and round the last bit. # first estimate the bits and do a big shift - mbits = w_m._count_bits() + mbits = b_mantissa._count_bits() needed = MANTISSA_BITS if mbits > needed: if mbits > needed+1: shifted = mbits - (needed+1) - w_m = rshift(space, w_m, mklong(space, shifted)) + b_mantissa = b_mantissa.rshift(shifted) bits += shifted # do the rounding bits += 1 - round = w_m.is_odd() - w_m = rshift(space, w_m, w_1) - w_m = adlong(space, w_m, mklong(space, round)) + round = b_mantissa.is_odd() + b_mantissa = b_mantissa.rshift(1).add(rbigint.fromint(round)) try: - r = math.ldexp(w_m.tofloat(), bits) + r = math.ldexp(b_mantissa.tofloat(), bits) # XXX I guess we do not check for overflow in ldexp as we agreed to! if r == 2*r and r != 0.0: raise OverflowError Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_complexobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_complexobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_complexobject.py Sun Dec 12 18:11:19 2010 @@ -1,5 +1,6 @@ import py -from pypy.objspace.std import complexobject as cobj +from pypy.objspace.std.complexobject import W_ComplexObject, \ + pow__Complex_Complex_ANY from pypy.objspace.std import complextype as cobjtype from pypy.objspace.std.multimethod import FailedToImplement from pypy.objspace.std.stringobject import W_StringObject @@ -11,7 +12,7 @@ def _test_instantiation(self): def _t_complex(r=0.0,i=0.0): - c = cobj.W_ComplexObject(r, i) + c = W_ComplexObject(r, i) assert c.real == float(r) and c.imag == float(i) pairs = ( (1, 1), @@ -41,21 +42,31 @@ def test_pow(self): - assert cobj._pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) - assert cobj._pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) - rr, ir = cobj._pow((0.0,1.0),(2.0,0.0)) + def _pow((r1, i1), (r2, i2)): + w_res = W_ComplexObject(r1, i1).pow(W_ComplexObject(r2, i2)) + return w_res.realval, w_res.imagval + assert _pow((0.0,2.0),(0.0,0.0)) == (1.0,0.0) + assert _pow((0.0,0.0),(2.0,0.0)) == (0.0,0.0) + rr, ir = _pow((0.0,1.0),(2.0,0.0)) assert abs(-1.0 - rr) < EPS assert abs(0.0 - ir) < EPS - assert cobj._powu((0.0,2.0),0) == (1.0,0.0) - assert cobj._powu((0.0,0.0),2) == (0.0,0.0) - assert cobj._powu((0.0,1.0),2) == (-1.0,0.0) - assert cobj._powi((0.0,2.0),0) == (1.0,0.0) - assert cobj._powi((0.0,0.0),2) == (0.0,0.0) - assert cobj._powi((0.0,1.0),2) == (-1.0,0.0) - c = cobj.W_ComplexObject(0.0,1.0) - p = cobj.W_ComplexObject(2.0,0.0) - r = cobj.pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) + def _powu((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_positive_int(n) + return w_res.realval, w_res.imagval + assert _powu((0.0,2.0),0) == (1.0,0.0) + assert _powu((0.0,0.0),2) == (0.0,0.0) + assert _powu((0.0,1.0),2) == (-1.0,0.0) + + def _powi((r1, i1), n): + w_res = W_ComplexObject(r1, i1).pow_int(n) + return w_res.realval, w_res.imagval + assert _powi((0.0,2.0),0) == (1.0,0.0) + assert _powi((0.0,0.0),2) == (0.0,0.0) + assert _powi((0.0,1.0),2) == (-1.0,0.0) + c = W_ComplexObject(0.0,1.0) + p = W_ComplexObject(2.0,0.0) + r = pow__Complex_Complex_ANY(self.space,c,p,self.space.wrap(None)) assert r.realval == -1.0 assert r.imagval == 0.0 Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_strutil.py Sun Dec 12 18:11:19 2010 @@ -134,8 +134,6 @@ assert string_to_bigint('1891234174197319').tolong() == 1891234174197319 def test_string_to_float(self): - def string_to_float(x): - return interp_string_to_float(self.space, x) assert string_to_float('0') == 0.0 assert string_to_float('1') == 1.0 assert string_to_float('-1.5') == -1.5 @@ -183,3 +181,4 @@ print repr(s) if s.strip(): # empty s raises OperationError directly py.test.raises(ParseStringError, string_to_float, s) + py.test.raises(ParseStringError, string_to_float, "") Modified: pypy/branch/fast-forward/pypy/rlib/rmd5.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rmd5.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rmd5.py Sun Dec 12 18:11:19 2010 @@ -29,20 +29,30 @@ if r_uint.BITS == 32: - def _mask(x): - "No-op on 32-bit platforms." - return x -else: - def _mask(x): - "Masks the r_uint value x to keep only the lowest 32 bits." - return x & r_uint(0xffffffff) - + def _rotateLeft(x, n): + "Rotate x (32 bit) left n bits circularly." + return (x << n) | (x >> (32-n)) -def _rotateLeft(x, n): - "Rotate x (32 bit) left n bits circularly." - - x = _mask(x) - return (x << n) | (x >> (32-n)) +else: + def _rotateLeft_emulator(x, n): + x &= 0xFFFFFFFF + return (x << n) | (x >> (32-n)) + + # ----- start of custom code, think about something better... ----- + from pypy.rpython.lltypesystem import lltype, rffi + from pypy.translator.tool.cbuild import ExternalCompilationInfo + eci = ExternalCompilationInfo(post_include_bits=[""" +static unsigned long pypy__rotateLeft(unsigned long x, long n) { + unsigned int x1 = x; /* arithmetic directly on int */ + int n1 = n; + return (x1 << n1) | (x1 >> (32-n1)); +} +"""]) + _rotateLeft = rffi.llexternal( + "pypy__rotateLeft", [lltype.Unsigned, lltype.Signed], lltype.Unsigned, + _callable=_rotateLeft_emulator, compilation_info=eci, + _nowrapper=True, pure_function=True) + # we expect the function _rotateLeft to be actually inlined def _state2string(a, b, c, d): Modified: pypy/branch/fast-forward/pypy/rlib/rmmap.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rmmap.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rmmap.py Sun Dec 12 18:11:19 2010 @@ -67,7 +67,7 @@ constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY', 'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY', 'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE', - 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE'] + 'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS'] for name in constant_names: setattr(CConfig, name, rffi_platform.ConstantInteger(name)) @@ -100,8 +100,11 @@ sandboxsafe=True, threadsafe=False) return unsafe, safe -def winexternal(name, args, result): - return rffi.llexternal(name, args, result, compilation_info=CConfig._compilation_info_, calling_conv='win') +def winexternal(name, args, result, **kwargs): + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_, + calling_conv='win', + **kwargs) PTR = rffi.CCHARP @@ -190,9 +193,17 @@ VirtualAlloc = winexternal('VirtualAlloc', [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD], rffi.VOIDP) - VirtualProtect = winexternal('VirtualProtect', - [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], - BOOL) + # VirtualProtect is used in llarena and should not release the GIL + _VirtualProtect = winexternal('VirtualProtect', + [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], + BOOL, + _nowrapper=True) + def VirtualProtect(addr, size, mode, oldmode_ptr): + return _VirtualProtect(addr, + rffi.cast(rffi.SIZE_T, size), + rffi.cast(DWORD, mode), + oldmode_ptr) + VirtualProtect._annspecialcase_ = 'specialize:ll' VirtualFree = winexternal('VirtualFree', [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL) Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/llarena.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/llarena.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/llarena.py Sun Dec 12 18:11:19 2010 @@ -26,6 +26,7 @@ self.objectptrs = {} # {offset: ptr-to-container} self.objectsizes = {} # {offset: size} self.freed = False + self.protect_inaccessible = None self.reset(zero) def __repr__(self): @@ -59,6 +60,8 @@ def check(self): if self.freed: raise ArenaError("arena was already freed") + if self.protect_inaccessible is not None: + raise ArenaError("arena is currently arena_protect()ed") def _getid(self): address, length = self.usagemap.buffer_info() @@ -127,6 +130,21 @@ def mark_freed(self): self.freed = True # this method is a hook for tests + def set_protect(self, inaccessible): + if inaccessible: + assert self.protect_inaccessible is None + saved = [] + for ptr in self.objectptrs.values(): + obj = ptr._obj + saved.append((obj, obj._protect())) + self.protect_inaccessible = saved + else: + assert self.protect_inaccessible is not None + saved = self.protect_inaccessible + for obj, storage in saved: + obj._unprotect(storage) + self.protect_inaccessible = None + class fakearenaaddress(llmemory.fakeaddress): def __init__(self, arena, offset): @@ -365,6 +383,16 @@ """ return Arena(ptr.arena.nbytes, False).getaddr(0) +def arena_protect(arena_addr, size, inaccessible): + """For debugging, set or reset memory protection on an arena. + For now, the starting point and size should reference the whole arena. + The value of 'inaccessible' is a boolean. + """ + arena_addr = getfakearenaaddress(arena_addr) + assert arena_addr.offset == 0 + assert size == arena_addr.arena.nbytes + arena_addr.arena.set_protect(inaccessible) + # ____________________________________________________________ # # Translation support: the functions above turn into the code below. @@ -475,6 +503,42 @@ # them immediately. clear_large_memory_chunk = llmemory.raw_memclear +if os.name == "posix": + from pypy.translator.tool.cbuild import ExternalCompilationInfo + _eci = ExternalCompilationInfo(includes=['sys/mman.h']) + raw_mprotect = rffi.llexternal('mprotect', + [llmemory.Address, rffi.SIZE_T, rffi.INT], + rffi.INT, + sandboxsafe=True, _nowrapper=True, + compilation_info=_eci) + def llimpl_protect(addr, size, inaccessible): + if inaccessible: + prot = 0 + else: + from pypy.rlib.rmmap import PROT_READ, PROT_WRITE + prot = PROT_READ | PROT_WRITE + raw_mprotect(addr, rffi.cast(rffi.SIZE_T, size), + rffi.cast(rffi.INT, prot)) + # ignore potential errors + has_protect = True + +elif os.name == 'nt': + def llimpl_protect(addr, size, inaccessible): + from pypy.rlib.rmmap import VirtualProtect, LPDWORD + if inaccessible: + from pypy.rlib.rmmap import PAGE_NOACCESS as newprotect + else: + from pypy.rlib.rmmap import PAGE_READWRITE as newprotect + arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') + VirtualProtect(rffi.cast(rffi.VOIDP, addr), + size, newprotect, arg) + # ignore potential errors + lltype.free(arg, flavor='raw') + has_protect = True + +else: + has_protect = False + llimpl_malloc = rffi.llexternal('malloc', [lltype.Signed], llmemory.Address, sandboxsafe=True, _nowrapper=True) @@ -544,6 +608,21 @@ 'll_arena.arena_new_view', llimpl=llimpl_arena_new_view, llfakeimpl=arena_new_view, sandboxsafe=True) +def llimpl_arena_protect(addr, size, inaccessible): + if has_protect: + # do some alignment + start = rffi.cast(lltype.Signed, addr) + end = start + size + start = (start + 4095) & ~ 4095 + end = end & ~ 4095 + if end > start: + llimpl_protect(rffi.cast(llmemory.Address, start), end-start, + inaccessible) +register_external(arena_protect, [llmemory.Address, lltype.Signed, + lltype.Bool], lltype.Void, + 'll_arena.arena_protect', llimpl=llimpl_arena_protect, + llfakeimpl=arena_protect, sandboxsafe=True) + def llimpl_getfakearenaaddress(addr): return addr register_external(getfakearenaaddress, [llmemory.Address], llmemory.Address, Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/lltype.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/lltype.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/lltype.py Sun Dec 12 18:11:19 2010 @@ -1388,6 +1388,15 @@ self._check() # no double-frees self._storage = None + def _protect(self): + result = self._storage + self._free() # no double-frees or double-protects + return result + + def _unprotect(self, saved_storage): + assert self._storage is None + self._storage = saved_storage + def _was_freed(self): if self._storage is None: return True Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/rstr.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/rstr.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/rstr.py Sun Dec 12 18:11:19 2010 @@ -242,10 +242,7 @@ FAST_RFIND = 2 -# XXX: This should be set to the number of bits in a long. Having a lower -# value here doesn't break anything, it just decreases the accuracy of the -# bloom filter heuristic, which results in a worse runtime (but correct results) -BLOOM_WIDTH = 32 +from pypy.rlib.rarithmetic import LONG_BIT as BLOOM_WIDTH def bloom_add(mask, c): Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_llarena.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_llarena.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/test/test_llarena.py Sun Dec 12 18:11:19 2010 @@ -6,6 +6,8 @@ from pypy.rpython.lltypesystem.llarena import round_up_for_allocation from pypy.rpython.lltypesystem.llarena import ArenaError, arena_new_view from pypy.rpython.lltypesystem.llarena import arena_shrink_obj +from pypy.rpython.lltypesystem.llarena import arena_protect, has_protect +from pypy.translator.c.test import test_genc, test_standalone def test_arena(): S = lltype.Struct('S', ('x',lltype.Signed)) @@ -265,8 +267,7 @@ assert res == 42 def test_compiled(): - from pypy.translator.c.test.test_genc import compile - fn = compile(test_look_inside_object, []) + fn = test_genc.compile(test_look_inside_object, []) res = fn() assert res == 42 @@ -282,3 +283,51 @@ arena_reserve(a, size_gc_header + llmemory.sizeof(S, 10)) arena_shrink_obj(a, size_gc_header + llmemory.sizeof(S, 5)) arena_reset(a, size_gc_header + llmemory.sizeof(S, 5), False) + +def test_arena_protect(): + a = arena_malloc(100, False) + S = lltype.Struct('S', ('x', lltype.Signed)) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 100, True) + py.test.raises(ArenaError, arena_reserve, a + 48, llmemory.sizeof(S)) + py.test.raises(RuntimeError, "p.x") + py.test.raises(RuntimeError, "p.x = 124") + arena_protect(a, 100, False) + assert p.x == 123 + p.x = 125 + assert p.x == 125 + + +class TestStandalone(test_standalone.StandaloneTests): + def test_compiled_arena_protect(self): + import os + from pypy.translator.c.test.test_genc import compile + S = lltype.Struct('S', ('x', lltype.Signed)) + # + def fn(argv): + testrun = int(argv[1]) + a = arena_malloc(65536, False) + arena_reserve(a, llmemory.sizeof(S)) + p = llmemory.cast_adr_to_ptr(a + 23432, lltype.Ptr(S)) + p.x = 123 + assert p.x == 123 + arena_protect(a, 65536, True) + result = 0 + if testrun == 1: + print p.x # segfault + if testrun == 2: + p.x = 124 # segfault + arena_protect(a, 65536, False) + p.x += 10 + print p.x + return 0 + # + t, cbuilder = self.compile(fn) + data = cbuilder.cmdexec('0') + assert data == '133\n' + if has_protect: + cbuilder.cmdexec('1', expect_crash=True) + cbuilder.cmdexec('2', expect_crash=True) Modified: pypy/branch/fast-forward/pypy/rpython/memory/gc/base.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gc/base.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gc/base.py Sun Dec 12 18:11:19 2010 @@ -36,6 +36,12 @@ self.finalizer_lock_count = 0 self.run_finalizers = self.AddressDeque() + def post_setup(self): + # More stuff that needs to be initialized when the GC is already + # fully working. (Only called by gctransform/framework for now.) + from pypy.rpython.memory.gc import env + self.DEBUG = env.read_from_env('PYPY_GC_DEBUG') + def _teardown(self): pass @@ -48,7 +54,8 @@ # The following flag enables costly consistency checks after each # collection. It is automatically set to True by test_gc.py. The # checking logic is translatable, so the flag can be set to True - # here before translation. + # here before translation. At run-time, if PYPY_GC_DEBUG is set, + # then it is also set to True. DEBUG = False def set_query_functions(self, is_varsize, has_gcptr_in_varsize, Modified: pypy/branch/fast-forward/pypy/rpython/memory/gc/env.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gc/env.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gc/env.py Sun Dec 12 18:11:19 2010 @@ -4,7 +4,7 @@ import os, sys from pypy.rlib.rarithmetic import r_uint from pypy.rlib.debug import debug_print, debug_start, debug_stop -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.lltypesystem.lloperation import llop # ____________________________________________________________ @@ -110,15 +110,9 @@ # ____________________________________________________________ # Estimation of the nursery size, based on the L2 cache. -def best_nursery_size_for_L2cache(L2cache): - # Heuristically, the best nursery size to choose is about half - # of the L2 cache. XXX benchmark some more. - if L2cache > 0: - return L2cache // 2 - else: - return -1 +# ---------- Linux2 ---------- -def get_L2cache_linux2(filename): +def get_L2cache_linux2(filename="/proc/cpuinfo"): debug_start("gc-hardware") L2cache = sys.maxint try: @@ -189,61 +183,66 @@ pos += 1 return pos +# ---------- Darwin ---------- -if sys.platform == 'linux2': - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. Linux code.""" - L2cache = get_L2cache_linux2('/proc/cpuinfo') - return best_nursery_size_for_L2cache(L2cache) - -elif sys.platform == 'darwin': - from pypy.rpython.lltypesystem import rffi - - sysctlbyname = rffi.llexternal('sysctlbyname', - [rffi.CCHARP, rffi.VOIDP, rffi.SIZE_TP, - rffi.VOIDP, rffi.SIZE_T], - rffi.INT, - sandboxsafe=True) - - def estimate_best_nursery_size(): - """Try to estimate the best nursery size at run-time, depending - on the machine we are running on. - """ - debug_start("gc-hardware") - L2cache = 0 - l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') +sysctlbyname = rffi.llexternal('sysctlbyname', + [rffi.CCHARP, rffi.VOIDP, rffi.SIZE_TP, + rffi.VOIDP, rffi.SIZE_T], + rffi.INT, + sandboxsafe=True) +def get_L2cache_darwin(): + """Try to estimate the best nursery size at run-time, depending + on the machine we are running on. + """ + debug_start("gc-hardware") + L2cache = 0 + l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') + try: + len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') try: - len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') - try: - size = rffi.sizeof(rffi.LONGLONG) - l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) - len_p[0] = rffi.cast(rffi.SIZE_T, size) - # XXX a hack for llhelper not being robust-enough - result = sysctlbyname("hw.l2cachesize", - rffi.cast(rffi.VOIDP, l2cache_p), - len_p, - lltype.nullptr(rffi.VOIDP.TO), - rffi.cast(rffi.SIZE_T, 0)) - if (rffi.cast(lltype.Signed, result) == 0 and - rffi.cast(lltype.Signed, len_p[0]) == size): - L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) - if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: - L2cache = 0 # overflow! - finally: - lltype.free(len_p, flavor='raw') + size = rffi.sizeof(rffi.LONGLONG) + l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) + len_p[0] = rffi.cast(rffi.SIZE_T, size) + # XXX a hack for llhelper not being robust-enough + result = sysctlbyname("hw.l2cachesize", + rffi.cast(rffi.VOIDP, l2cache_p), + len_p, + lltype.nullptr(rffi.VOIDP.TO), + rffi.cast(rffi.SIZE_T, 0)) + if (rffi.cast(lltype.Signed, result) == 0 and + rffi.cast(lltype.Signed, len_p[0]) == size): + L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) + if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: + L2cache = 0 # overflow! finally: - lltype.free(l2cache_p, flavor='raw') - debug_print("L2cache =", L2cache) - debug_stop("gc-hardware") - if L2cache > 0: - return L2cache - else: - # Print a top-level warning even in non-debug builds - llop.debug_print(lltype.Void, - "Warning: cannot find your CPU L2 cache size with sysctl()") - return -1 + lltype.free(len_p, flavor='raw') + finally: + lltype.free(l2cache_p, flavor='raw') + debug_print("L2cache =", L2cache) + debug_stop("gc-hardware") + if L2cache > 0: + return L2cache + else: + # Print a top-level warning even in non-debug builds + llop.debug_print(lltype.Void, + "Warning: cannot find your CPU L2 cache size with sysctl()") + return -1 -else: - def estimate_best_nursery_size(): - return -1 # XXX implement me for other platforms +# -------------------- + +get_L2cache = globals().get('get_L2cache_' + sys.platform, + lambda: -1) # implement me for other platforms + +def best_nursery_size_for_L2cache(L2cache): + # Heuristically, the best nursery size to choose is about half + # of the L2 cache. + if L2cache > 0: + return L2cache // 2 + else: + return -1 + +def estimate_best_nursery_size(): + """Try to estimate the best nursery size at run-time, depending + on the machine we are running on. Linux code.""" + L2cache = get_L2cache() + return best_nursery_size_for_L2cache(L2cache) Modified: pypy/branch/fast-forward/pypy/rpython/memory/gc/minimark.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gc/minimark.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gc/minimark.py Sun Dec 12 18:11:19 2010 @@ -3,7 +3,8 @@ Environment variables can be used to fine-tune the following parameters: PYPY_GC_NURSERY The nursery size. Defaults to half the size of - the L2 cache. Try values like '1.2MB'. + the L2 cache. Try values like '1.2MB'. Small values + (like 1 or 1KB) are useful for debugging. PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', which means trigger a major collection when the @@ -32,6 +33,11 @@ limit. Useful to avoid spending all the time in the GC in very small programs. Defaults to 8 times the nursery. + + PYPY_GC_DEBUG Enable extra checks around collections that are + too slow for normal use. Values are 0 (off), + 1 (on major collections) or 2 (also on minor + collections). """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -223,7 +229,8 @@ self.nursery = NULL self.nursery_free = NULL self.nursery_top = NULL - self.debug_always_do_minor_collect = False + self.debug_tiny_nursery = -1 + self.debug_rotating_nurseries = None # # The ArenaCollection() handles the nonmovable objects allocation. if ArenaCollectionClass is None: @@ -291,15 +298,20 @@ # From there on, the GC is fully initialized and the code # below can use it newsize = env.read_from_env('PYPY_GC_NURSERY') - # PYPY_GC_NURSERY=1 forces a minor collect for every malloc. - # Useful to debug external factors, like trackgcroot or the - # handling of the write barrier. - self.debug_always_do_minor_collect = newsize == 1 + # PYPY_GC_NURSERY=smallvalue means that minor collects occur + # very frequently; the extreme case is PYPY_GC_NURSERY=1, which + # forces a minor collect for every malloc. Useful to debug + # external factors, like trackgcroot or the handling of the write + # barrier. Implemented by still using 'minsize' for the nursery + # size (needed to handle e.g. mallocs of 8249 words) but hacking + # at the current nursery position in collect_and_reserve(). if newsize <= 0: newsize = env.estimate_best_nursery_size() if newsize <= 0: newsize = defaultsize - newsize = max(newsize, minsize) + if newsize < minsize: + self.debug_tiny_nursery = newsize & ~(WORD-1) + newsize = minsize # major_coll = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT') if major_coll > 1.0: @@ -332,17 +344,24 @@ self.allocate_nursery() - def allocate_nursery(self): - debug_start("gc-set-nursery-size") - debug_print("nursery size:", self.nursery_size) + def _nursery_memory_size(self): + extra = self.nonlarge_gcptrs_max + 1 + return self.nursery_size + extra + + def _alloc_nursery(self): # the start of the nursery: we actually allocate a bit more for # the nursery than really needed, to simplify pointer arithmetic # in malloc_fixedsize_clear(). The few extra pages are never used # anyway so it doesn't even count. - extra = self.nonlarge_gcptrs_max + 1 - self.nursery = llarena.arena_malloc(self.nursery_size + extra, 2) - if not self.nursery: + nursery = llarena.arena_malloc(self._nursery_memory_size(), 2) + if not nursery: raise MemoryError("cannot allocate nursery") + return nursery + + def allocate_nursery(self): + debug_start("gc-set-nursery-size") + debug_print("nursery size:", self.nursery_size) + self.nursery = self._alloc_nursery() # the current position in the nursery: self.nursery_free = self.nursery # the end of the nursery: @@ -376,6 +395,40 @@ return bounded + def post_setup(self): + # set up extra stuff for PYPY_GC_DEBUG. + MovingGCBase.post_setup(self) + if self.DEBUG and llarena.has_protect: + # gc debug mode: allocate 23 nurseries instead of just 1, + # and use them alternatively, while mprotect()ing the unused + # ones to detect invalid access. + debug_start("gc-debug") + self.debug_rotating_nurseries = [] + for i in range(22): + nurs = self._alloc_nursery() + llarena.arena_protect(nurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(nurs) + debug_print("allocated", len(self.debug_rotating_nurseries), + "extra nurseries") + debug_stop("gc-debug") + + def debug_rotate_nursery(self): + if self.debug_rotating_nurseries is not None: + debug_start("gc-debug") + oldnurs = self.nursery + llarena.arena_protect(oldnurs, self._nursery_memory_size(), True) + self.debug_rotating_nurseries.append(oldnurs) + # + newnurs = self.debug_rotating_nurseries.pop(0) + llarena.arena_protect(newnurs, self._nursery_memory_size(), False) + self.nursery = newnurs + self.nursery_top = self.nursery + self.nursery_size + debug_print("switching from nursery", oldnurs, + "to nursery", self.nursery, + "size", self.nursery_size) + debug_stop("gc-debug") + + def malloc_fixedsize_clear(self, typeid, size, can_collect=True, needs_finalizer=False, contains_weakptr=False): ll_assert(can_collect, "!can_collect") @@ -513,8 +566,9 @@ self.nursery_free = result + totalsize ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow") # - if self.debug_always_do_minor_collect: - self.nursery_free = self.nursery_top + if self.debug_tiny_nursery >= 0: # for debugging + if self.nursery_top - self.nursery_free > self.debug_tiny_nursery: + self.nursery_free = self.nursery_top - self.debug_tiny_nursery # return result collect_and_reserve._dont_inline_ = True @@ -841,7 +895,7 @@ def remember_young_pointer(addr_struct, newvalue): # 'addr_struct' is the address of the object in which we write. # 'newvalue' is the address that we are going to write in there. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_struct), "nursery object with GCFLAG_NO_YOUNG_PTRS") # @@ -878,7 +932,7 @@ # 'addr_array' is the address of the object in which we write, # which must have an array part; 'index' is the index of the # item that is (or contains) the pointer that we write. - if DEBUG: + if DEBUG: # note: PYPY_GC_DEBUG=1 does not enable this ll_assert(not self.is_in_nursery(addr_array), "nursery array with GCFLAG_NO_YOUNG_PTRS") objhdr = self.header(addr_array) @@ -1000,13 +1054,14 @@ # All live nursery objects are out, and the rest dies. Fill # the whole nursery with zero and reset the current nursery pointer. llarena.arena_reset(self.nursery, self.nursery_size, 2) + self.debug_rotate_nursery() self.nursery_free = self.nursery # debug_print("minor collect, total memory used:", self.get_total_memory_used()) + if self.DEBUG >= 2: + self.debug_check_consistency() # expensive! debug_stop("gc-minor") - if 0: # not we_are_translated(): - self.debug_check_consistency() # xxx expensive! def collect_roots_in_nursery(self): Modified: pypy/branch/fast-forward/pypy/rpython/memory/gctransform/asmgcroot.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gctransform/asmgcroot.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gctransform/asmgcroot.py Sun Dec 12 18:11:19 2010 @@ -306,6 +306,16 @@ if item: self._shape_decompressor.setaddr(item) return + # there is a rare risk that the array contains *two* entries + # with the same key, one of which is dead (null value), and we + # found the dead one above. Solve this case by replacing all + # dead keys with nulls, sorting again, and then trying again. + replace_dead_entries_with_nulls(gcmapstart2, gcmapend2) + sort_gcmap(gcmapstart2, gcmapend2) + item = search_in_gcmap2(gcmapstart2, gcmapend2, retaddr) + if item: + self._shape_decompressor.setaddr(item) + return # the item may have been not found because the main array was # not sorted. Sort it and try again. win32_follow_gcmap_jmp(gcmapstart, gcmapend) @@ -401,6 +411,15 @@ rffi.cast(rffi.SIZE_T, arrayitemsize), llhelper(QSORT_CALLBACK_PTR, _compare_gcmap_entries)) +def replace_dead_entries_with_nulls(start, end): + # replace the dead entries (null value) with a null key. + count = (end - start) // arrayitemsize - 1 + while count >= 0: + item = start + count * arrayitemsize + if item.address[1] == llmemory.NULL: + item.address[0] = llmemory.NULL + count -= 1 + if sys.platform == 'win32': def win32_follow_gcmap_jmp(start, end): # The initial gcmap table contains addresses to a JMP Modified: pypy/branch/fast-forward/pypy/rpython/memory/gctransform/framework.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/memory/gctransform/framework.py (original) +++ pypy/branch/fast-forward/pypy/rpython/memory/gctransform/framework.py Sun Dec 12 18:11:19 2010 @@ -189,6 +189,7 @@ # run-time initialization code root_walker.setup_root_walker() gcdata.gc.setup() + gcdata.gc.post_setup() def frameworkgc__teardown(): # run-time teardown code for tests! Modified: pypy/branch/fast-forward/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/fast-forward/pypy/rpython/module/ll_os.py Sun Dec 12 18:11:19 2010 @@ -730,6 +730,22 @@ return extdef([traits.str, int, int], int, traits.ll_os_name('open'), llimpl=os_open_llimpl, oofakeimpl=os_open_oofakeimpl) + @registering_if(os, 'getloadavg') + def register_os_getloadavg(self): + AP = rffi.CArrayPtr(lltype.Float) + c_getloadavg = self.llexternal('getloadavg', [AP, rffi.INT], rffi.INT) + + def getloadavg_llimpl(): + load = lltype.malloc(AP.TO, 3, flavor='raw') + r = c_getloadavg(load, 3) + result_tuple = load[0], load[1], load[2] + lltype.free(load, flavor='raw') + if r != 3: + raise OSError + return result_tuple + return extdef([], (float, float, float), + "ll_os.ll_getloadavg", llimpl=getloadavg_llimpl) + # ------------------------------- os.read ------------------------------- @registering(os.read) @@ -887,6 +903,18 @@ llimpl=fdatasync_llimpl, export_name="ll_os.ll_os_fdatasync") + @registering_if(os, 'fchdir') + def register_os_fchdir(self): + os_fchdir = self.llexternal('fchdir', [rffi.INT], rffi.INT) + + def fchdir_llimpl(fd): + res = rffi.cast(rffi.LONG, os_fchdir(rffi.cast(rffi.INT, fd))) + if res < 0: + raise OSError(rposix.get_errno(), "fchdir failed") + return extdef([int], s_None, + llimpl=fchdir_llimpl, + export_name="ll_os.ll_os_fchdir") + @registering_str_unicode(os.access) def register_os_access(self, traits): os_access = self.llexternal(traits.posix_function_name('access'), Modified: pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py Sun Dec 12 18:11:19 2010 @@ -755,3 +755,31 @@ for i in range(5): res = func(i) assert res == os.uname()[i] + +if hasattr(os, 'getloadavg'): + def test_os_getloadavg(): + def does_stuff(): + a, b, c = os.getloadavg() + print a, b, c + return a + b + c + f = compile(does_stuff, []) + res = f() + assert type(res) is float and res >= 0.0 + +if hasattr(os, 'fchdir'): + def test_os_fchdir(): + def does_stuff(): + fd = os.open('/', os.O_RDONLY, 0400) + try: + os.fchdir(fd) + s = os.getcwd() + finally: + os.close(fd) + return s == '/' + f = compile(does_stuff, []) + localdir = os.getcwd() + try: + res = f() + finally: + os.chdir(localdir) + assert res == True Modified: pypy/branch/fast-forward/pypy/translator/sandbox/test/test_sandbox.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/sandbox/test/test_sandbox.py (original) +++ pypy/branch/fast-forward/pypy/translator/sandbox/test/test_sandbox.py Sun Dec 12 18:11:19 2010 @@ -148,6 +148,7 @@ if sys.platform == 'linux2': # on Mac, uses another (sandboxsafe) approach expect(f, g, "ll_os.ll_os_open", ("/proc/cpuinfo", 0, 420), OSError(5232, "xyz")) + expect(f, g, "ll_os.ll_os_getenv", ("PYPY_GC_DEBUG",), None) g.close() tail = f.read() f.close() From arigo at codespeak.net Sun Dec 12 19:55:21 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Sun, 12 Dec 2010 19:55:21 +0100 (CET) Subject: [pypy-svn] r80005 - in pypy/branch/jit-stackcheck/pypy: rlib rpython/lltypesystem translator/c/src Message-ID: <20101212185521.B485D282B90@codespeak.net> Author: arigo Date: Sun Dec 12 19:55:19 2010 New Revision: 80005 Modified: pypy/branch/jit-stackcheck/pypy/rlib/rstack.py pypy/branch/jit-stackcheck/pypy/rpython/lltypesystem/lloperation.py pypy/branch/jit-stackcheck/pypy/translator/c/src/stack.h Log: Rewrite the stack overflow detection logic. The goal is to get a simple check done in rlib/rstack.py, followed by a call to the slow path if it fails; the check should be simple enough to be easily inlinable by the JIT. Modified: pypy/branch/jit-stackcheck/pypy/rlib/rstack.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/rlib/rstack.py (original) +++ pypy/branch/jit-stackcheck/pypy/rlib/rstack.py Sun Dec 12 19:55:19 2010 @@ -6,21 +6,21 @@ import inspect from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rarithmetic import r_uint from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.controllerentry import Controller, SomeControlledInstance from pypy.translator.tool.cbuild import ExternalCompilationInfo def stack_unwind(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_unwind(lltype.Void) raise RuntimeError("cannot unwind stack in non-translated versions") def stack_capture(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop ptr = llop.stack_capture(OPAQUE_STATE_HEADER_PTR) return frame_stack_top_controller.box(ptr) raise RuntimeError("cannot unwind stack in non-translated versions") @@ -28,22 +28,48 @@ def stack_frames_depth(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_frames_depth(lltype.Signed) else: return len(inspect.stack()) +# ____________________________________________________________ + compilation_info = ExternalCompilationInfo(includes=['src/stack.h']) -stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, - compilation_info=compilation_info, - _nowrapper=True, - _callable=lambda: _zero, - sandboxsafe=True) -_zero = rffi.cast(rffi.INT, 0) +def llexternal(name, args, res): + return rffi.llexternal(name, args, res, compilation_info=compilation_info, + sandboxsafe=True, _nowrapper=True) + +_stack_get_start = llexternal('LL_stack_get_start', [], lltype.Signed) +_stack_get_length = llexternal('LL_stack_get_length', [], lltype.Signed) +_stack_too_big_slowpath = llexternal('LL_stack_too_big_slowpath', + [lltype.Signed], lltype.Char) +# the following is used by the JIT +_stack_get_start_adr = llexternal('LL_stack_get_start_adr', [], lltype.Signed) + def stack_check(): - if rffi.cast(lltype.Signed, stack_too_big()): + if not we_are_translated(): + return + # + # Load the "current" stack position, or at least some address that + # points close to the current stack head + current = llop.stack_current(lltype.Signed) + # + # Load these variables from C code + start = _stack_get_start() + length = _stack_get_length() + # + # Common case: if 'current' is within [start:start+length], everything + # is fine + ofs = r_uint(current - start) + if ofs < r_uint(length): + return + # + # Else call the slow path + if ord(_stack_too_big_slowpath(current)): + # + # Now we are sure that the stack is really too big. Note that the # stack_unwind implementation is different depending on if stackless # is enabled. If it is it unwinds the stack, otherwise it simply # raises a RuntimeError. Modified: pypy/branch/jit-stackcheck/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/jit-stackcheck/pypy/rpython/lltypesystem/lloperation.py Sun Dec 12 19:55:19 2010 @@ -526,6 +526,8 @@ 'get_stack_depth_limit':LLOp(sideeffects=False), 'set_stack_depth_limit':LLOp(), + 'stack_current': LLOp(sideeffects=False), + # __________ misc operations __________ 'keepalive': LLOp(), Modified: pypy/branch/jit-stackcheck/pypy/translator/c/src/stack.h ============================================================================== --- pypy/branch/jit-stackcheck/pypy/translator/c/src/stack.h (original) +++ pypy/branch/jit-stackcheck/pypy/translator/c/src/stack.h Sun Dec 12 19:55:19 2010 @@ -11,27 +11,17 @@ * It is needed to have RPyThreadStaticTLS, too. */ #include "thread.h" +extern char *_LLstacktoobig_stack_start; + void LL_stack_unwind(void); -int LL_stack_too_big_slowpath(void); +char LL_stack_too_big_slowpath(long); /* returns 0 (ok) or 1 (too big) */ -extern volatile char *_LLstacktoobig_stack_base_pointer; -extern long _LLstacktoobig_stack_min; -extern long _LLstacktoobig_stack_max; +/* some macros referenced from pypy.rlib.rstack */ +#define OP_STACK_CURRENT(r) r = (long)&r +#define LL_stack_get_start() ((long)_LLstacktoobig_stack_start) +#define LL_stack_get_length() MAX_STACK_SIZE +#define LL_stack_get_start_adr() ((long)&_LLstacktoobig_stack_start) /* JIT */ -static int LL_stack_too_big(void) -{ - /* The fast path of stack_too_big, called extremely often. - Making it static makes an *inlinable* copy of this small - function's implementation in each compilation unit. */ - char local; - long diff = &local - _LLstacktoobig_stack_base_pointer; - /* common case: we are still in the same thread as last time - we checked, and still in the allowed part of the stack */ - return ((diff < _LLstacktoobig_stack_min || - diff > _LLstacktoobig_stack_max) - /* if not, call the slow path */ - && LL_stack_too_big_slowpath()); -} #ifdef __GNUC__ # define PYPY_INHIBIT_TAIL_CALL() asm("/* inhibit_tail_call */") @@ -61,68 +51,75 @@ return &local - parent; } -volatile char *_LLstacktoobig_stack_base_pointer = NULL; -long _LLstacktoobig_stack_min = 0; -long _LLstacktoobig_stack_max = 0; -RPyThreadStaticTLS _LLstacktoobig_stack_base_pointer_key; +char *_LLstacktoobig_stack_start = NULL; +int stack_direction = 0; +RPyThreadStaticTLS start_tls_key; -int LL_stack_too_big_slowpath(void) +char LL_stack_too_big_slowpath(long current) { - char local; long diff; - char *baseptr; - /* Check that the stack is less than MAX_STACK_SIZE bytes bigger - than the value recorded in stack_base_pointer. The base - pointer is updated to the current value if it is still NULL - or if we later find a &local that is below it. The real - stack base pointer is stored in thread-local storage, but we - try to minimize its overhead by keeping a local copy in - stack_pointer_pointer. */ + char *baseptr, *curptr = (char*)current; + + /* The stack_start variable is updated to match the current value + if it is still 0 or if we later find a 'curptr' position + that is below it. The real stack_start pointer is stored in + thread-local storage, but we try to minimize its overhead by + keeping a local copy in _LLstacktoobig_stack_start. */ - if (_LLstacktoobig_stack_min == _LLstacktoobig_stack_max /* == 0 */) { + if (stack_direction == 0) { /* not initialized */ /* XXX We assume that initialization is performed early, when there is still only one thread running. This allows us to ignore race conditions here */ - char *errmsg = RPyThreadStaticTLS_Create( - &_LLstacktoobig_stack_base_pointer_key); + char *errmsg = RPyThreadStaticTLS_Create(&start_tls_key); if (errmsg) { /* XXX should we exit the process? */ fprintf(stderr, "Internal PyPy error: %s\n", errmsg); return 1; } if (_LL_stack_growing_direction(NULL) > 0) - _LLstacktoobig_stack_max = MAX_STACK_SIZE; + stack_direction = +1; else - _LLstacktoobig_stack_min = -MAX_STACK_SIZE; + stack_direction = -1; } - baseptr = (char *) RPyThreadStaticTLS_Get( - _LLstacktoobig_stack_base_pointer_key); + baseptr = (char *) RPyThreadStaticTLS_Get(start_tls_key); if (baseptr != NULL) { - diff = &local - baseptr; - if (_LLstacktoobig_stack_min <= diff && - diff <= _LLstacktoobig_stack_max) { - /* within bounds */ - _LLstacktoobig_stack_base_pointer = baseptr; + diff = curptr - baseptr; + if (((unsigned long)diff) < (unsigned long)MAX_STACK_SIZE) { + /* within bounds, probably just had a thread switch */ + _LLstacktoobig_stack_start = baseptr; return 0; } - if ((_LLstacktoobig_stack_min == 0 && diff < 0) || - (_LLstacktoobig_stack_max == 0 && diff > 0)) { - /* we underflowed the stack, which means that - the initial estimation of the stack base must - be revised (see below) */ + if (stack_direction > 0) { + if (diff < 0 && diff > -MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } else { - return 1; /* stack overflow */ + if (diff >= MAX_STACK_SIZE && diff < 2*MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } + /* else we underflowed the stack, which means that + the initial estimation of the stack base must + be revised */ } /* update the stack base pointer to the current value */ - baseptr = &local; - RPyThreadStaticTLS_Set(_LLstacktoobig_stack_base_pointer_key, baseptr); - _LLstacktoobig_stack_base_pointer = baseptr; + if (stack_direction > 0) { + /* the valid range is [curptr:curptr+MAX_STACK_SIZE] */ + baseptr = curptr; + } + else { + /* the valid range is [curptr-MAX_STACK_SIZE+1:curptr+1] */ + baseptr = curptr - MAX_STACK_SIZE + 1; + } + RPyThreadStaticTLS_Set(start_tls_key, baseptr); + _LLstacktoobig_stack_start = baseptr; return 0; } From fijal at codespeak.net Sun Dec 12 20:29:33 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 12 Dec 2010 20:29:33 +0100 (CET) Subject: [pypy-svn] r80007 - pypy/branch/out-of-line-guards/pypy/jit/metainterp Message-ID: <20101212192933.73F26282B90@codespeak.net> Author: fijal Date: Sun Dec 12 20:29:30 2010 New Revision: 80007 Added: pypy/branch/out-of-line-guards/pypy/jit/metainterp/memmgr.py Log: Simply copy memmgr.py here. Not sure what svn merge did Added: pypy/branch/out-of-line-guards/pypy/jit/metainterp/memmgr.py ============================================================================== --- (empty file) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/memmgr.py Sun Dec 12 20:29:30 2010 @@ -0,0 +1,79 @@ +import math +from pypy.rlib.rarithmetic import r_int64 +from pypy.rlib.debug import debug_start, debug_print, debug_stop +from pypy.rlib.objectmodel import we_are_translated + +# +# Logic to decide which loops are old and not used any more. +# +# All the long-lived references to LoopToken are weakrefs (see JitCell +# in warmstate.py), apart from the 'alive_loops' set in MemoryManager, +# which is the only (long-living) place that keeps them alive. If a +# loop was not called for long enough, then it is removed from +# 'alive_loops'. It will soon be freed by the GC. LoopToken.__del__ +# calls the method cpu.free_loop_and_bridges(). +# +# The alive_loops set is maintained using the notion of a global +# 'current generation' which is, in practice, the total number of loops +# and bridges produced so far. A LoopToken is declared "old" if its +# 'generation' field is much smaller than the current generation, and +# removed from the set. +# + +class MemoryManager(object): + + def __init__(self): + self.check_frequency = -1 + # NB. use of r_int64 to be extremely far on the safe side: + # this is increasing by one after each loop or bridge is + # compiled, and it must not overflow. If the backend implements + # complete freeing in cpu.free_loop_and_bridges(), then it may + # be possible to get arbitrary many of them just by waiting long + # enough. But in this day and age, you'd still never have the + # patience of waiting for a slowly-increasing 64-bit number to + # overflow :-) + self.current_generation = r_int64(1) + self.next_check = r_int64(-1) + self.alive_loops = {} + + def set_max_age(self, max_age, check_frequency=0): + if max_age <= 0: + self.next_check = r_int64(-1) + else: + self.max_age = max_age + if check_frequency <= 0: + check_frequency = int(math.sqrt(max_age)) + self.check_frequency = check_frequency + self.next_check = self.current_generation + 1 + + def next_generation(self): + self.current_generation += 1 + if self.current_generation == self.next_check: + self._kill_old_loops_now() + self.next_check = self.current_generation + self.check_frequency + + def keep_loop_alive(self, looptoken): + if looptoken.generation != self.current_generation: + looptoken.generation = self.current_generation + self.alive_loops[looptoken] = None + + def _kill_old_loops_now(self): + debug_start("jit-mem-collect") + oldtotal = len(self.alive_loops) + #print self.alive_loops.keys() + debug_print("Current generation:", self.current_generation) + debug_print("Loop tokens before:", oldtotal) + max_generation = self.current_generation - (self.max_age-1) + for looptoken in self.alive_loops.keys(): + if 0 <= looptoken.generation < max_generation: + del self.alive_loops[looptoken] + newtotal = len(self.alive_loops) + debug_print("Loop tokens freed: ", oldtotal - newtotal) + debug_print("Loop tokens left: ", newtotal) + #print self.alive_loops.keys() + if not we_are_translated() and oldtotal != newtotal: + looptoken = None + from pypy.rlib import rgc + # a single one is not enough for all tests :-( + rgc.collect(); rgc.collect(); rgc.collect() + debug_stop("jit-mem-collect") From agaynor at codespeak.net Sun Dec 12 20:38:04 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Sun, 12 Dec 2010 20:38:04 +0100 (CET) Subject: [pypy-svn] r80008 - pypy/branch/fast-forward/pypy/module/select Message-ID: <20101212193804.95715282B90@codespeak.net> Author: agaynor Date: Sun Dec 12 20:38:01 2010 New Revision: 80008 Modified: pypy/branch/fast-forward/pypy/module/select/interp_epoll.py Log: Fixed epoll after the merge. Modified: pypy/branch/fast-forward/pypy/module/select/interp_epoll.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/select/interp_epoll.py (original) +++ pypy/branch/fast-forward/pypy/module/select/interp_epoll.py Sun Dec 12 20:38:01 2010 @@ -7,7 +7,6 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec, ObjSpace, W_Root from pypy.interpreter.error import OperationError, wrap_oserror, operationerrfmt from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.module.select.interp_select import as_fd_w from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.tool import rffi_platform from pypy.rlib._rsocket_rffi import socketclose, FD_SETSIZE @@ -101,7 +100,7 @@ self.epfd = -1 def epoll_ctl(self, ctl, w_fd, eventmask, ignore_ebadf=False): - fd = as_fd_w(self.space, w_fd) + fd = self.space.c_filedescriptor_w(w_fd) with lltype.scoped_alloc(epoll_event) as ev: ev.c_events = rffi.cast(rffi.UINT, eventmask) rffi.setintfield(ev.c_data, 'c_fd', fd) From fijal at codespeak.net Sun Dec 12 20:48:50 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Sun, 12 Dec 2010 20:48:50 +0100 (CET) Subject: [pypy-svn] r80009 - pypy/branch/out-of-line-guards/pypy/config Message-ID: <20101212194850.98681282B90@codespeak.net> Author: fijal Date: Sun Dec 12 20:48:48 2010 New Revision: 80009 Added: pypy/branch/out-of-line-guards/pypy/config/support.py Log: Another missing file from mergex Added: pypy/branch/out-of-line-guards/pypy/config/support.py ============================================================================== --- (empty file) +++ pypy/branch/out-of-line-guards/pypy/config/support.py Sun Dec 12 20:48:48 2010 @@ -0,0 +1,21 @@ + +""" Some support code +""" + +import re, sys, os + +def detect_number_of_processors(filename_or_file='/proc/cpuinfo'): + if sys.platform != 'linux2': + return 1 # implement me + if os.environ.get('MAKEFLAGS'): + return 1 # don't override MAKEFLAGS. This will call 'make' without any '-j' option + try: + if isinstance(filename_or_file, str): + f = open(filename_or_file, "r") + else: + f = filename_or_file + return max([int(re.split('processor.*(\d+)', line)[1]) + for line in f.readlines() + if line.startswith('processor')]) + 1 + except: + return 1 # we really don't want to explode here, at worst we have 1 From afa at codespeak.net Sun Dec 12 22:08:58 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 12 Dec 2010 22:08:58 +0100 (CET) Subject: [pypy-svn] r80010 - pypy/branch/fast-forward/pypy/module/select Message-ID: <20101212210858.E5BE2282B90@codespeak.net> Author: afa Date: Sun Dec 12 22:08:55 2010 New Revision: 80010 Modified: pypy/branch/fast-forward/pypy/module/select/interp_select.py Log: Another fix after merge Modified: pypy/branch/fast-forward/pypy/module/select/interp_select.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/select/interp_select.py (original) +++ pypy/branch/fast-forward/pypy/module/select/interp_select.py Sun Dec 12 22:08:55 2010 @@ -24,7 +24,7 @@ register.unwrap_spec = ['self', ObjSpace, W_Root, int] def modify(self, space, w_fd, events): - fd = as_fd_w(space, w_fd) + fd = space.c_filedescriptor_w(w_fd) if fd not in self.fddict: raise wrap_oserror(space, OSError(errno.ENOENT, "poll.modify"), exception_name='w_IOError') From afa at codespeak.net Sun Dec 12 23:07:46 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 12 Dec 2010 23:07:46 +0100 (CET) Subject: [pypy-svn] r80011 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20101212220746.60B12282B90@codespeak.net> Author: afa Date: Sun Dec 12 23:07:45 2010 New Revision: 80011 Modified: pypy/branch/fast-forward/pypy/objspace/std/strutil.py pypy/branch/fast-forward/pypy/objspace/std/test/test_strutil.py Log: Update error message to better match CPython Modified: pypy/branch/fast-forward/pypy/objspace/std/strutil.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/strutil.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/strutil.py Sun Dec 12 23:07:45 2010 @@ -34,11 +34,9 @@ class NumberStringParser: def error(self): - if self.literal: - raise ParseStringError, 'invalid literal for %s(): %s' % (self.fname, self.literal) - else: - raise ParseStringError, 'empty string for %s()' % (self.fname,) - + raise ParseStringError("invalid literal for %s() with base %d: '%s'" % + (self.fname, self.base, self.literal)) + def __init__(self, s, literal, base, fname): self.literal = literal self.fname = fname Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_strutil.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_strutil.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_strutil.py Sun Dec 12 23:07:45 2010 @@ -87,6 +87,9 @@ raises(ParseStringError, string_to_int, '0x', 16) raises(ParseStringError, string_to_int, '-0x', 16) + exc = raises(ParseStringError, string_to_int, '') + assert exc.value.msg == "invalid literal for int() with base 10: ''" + def test_string_to_int_overflow(self): import sys space = self.space From afa at codespeak.net Sun Dec 12 23:12:21 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Sun, 12 Dec 2010 23:12:21 +0100 (CET) Subject: [pypy-svn] r80012 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101212221221.7EAED282B90@codespeak.net> Author: afa Date: Sun Dec 12 23:12:20 2010 New Revision: 80012 Added: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_optparse.py - copied, changed from r79877, pypy/branch/fast-forward/lib-python/2.7.0/test/test_optparse.py Log: Skip a test based on refcounting. Copied: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_optparse.py (from r79877, pypy/branch/fast-forward/lib-python/2.7.0/test/test_optparse.py) ============================================================================== --- pypy/branch/fast-forward/lib-python/2.7.0/test/test_optparse.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_optparse.py Sun Dec 12 23:12:20 2010 @@ -383,6 +383,7 @@ self.assertRaises(self.parser.remove_option, ('foo',), None, ValueError, "no such option 'foo'") + test_support.impl_detail("sys.getrefcount") def test_refleak(self): # If an OptionParser is carrying around a reference to a large # object, various cycles can prevent it from being GC'd in From afa at codespeak.net Mon Dec 13 00:19:34 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 13 Dec 2010 00:19:34 +0100 (CET) Subject: [pypy-svn] r80013 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101212231934.0E948282B90@codespeak.net> Author: afa Date: Mon Dec 13 00:19:32 2010 New Revision: 80013 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py Log: Unskip this test, It does not block any more. It fails for another reason, maybe our buffered streams are broken after a fork() Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py Mon Dec 13 00:19:32 2010 @@ -468,7 +468,6 @@ """ self._run_and_join(script) - @unittest.skip("FIXME: pypy should have an 'after_fork' hook") def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. From afa at codespeak.net Mon Dec 13 00:35:43 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 13 Dec 2010 00:35:43 +0100 (CET) Subject: [pypy-svn] r80014 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101212233543.69B18282B90@codespeak.net> Author: afa Date: Mon Dec 13 00:35:41 2010 New Revision: 80014 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py Log: Skip two tests really tied to CPython architecture Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py Mon Dec 13 00:35:41 2010 @@ -159,6 +159,7 @@ # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) # exposed at the Python level. This test relies on ctypes to get at it. + @test.test_support.cpython_only def test_PyThreadState_SetAsyncExc(self): try: import ctypes @@ -264,6 +265,7 @@ finally: threading._start_new_thread = _start_new_thread + @test.test_support.cpython_only def test_finalize_runnning_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for From afa at codespeak.net Mon Dec 13 00:44:10 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 13 Dec 2010 00:44:10 +0100 (CET) Subject: [pypy-svn] r80015 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101212234410.CEAEF282B90@codespeak.net> Author: afa Date: Mon Dec 13 00:44:08 2010 New Revision: 80015 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py Log: Don't try to "fix" this test with gc.collect(): the goal is to test how CPython deals with reference cycles in thread objects. Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_threading.py Mon Dec 13 00:44:08 2010 @@ -386,6 +386,7 @@ finally: sys.setcheckinterval(old_interval) + @test.test_support.cpython_only def test_no_refcycle_through_target(self): class RunSelfFunction(object): def __init__(self, should_raise): From afa at codespeak.net Mon Dec 13 01:03:06 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 13 Dec 2010 01:03:06 +0100 (CET) Subject: [pypy-svn] r80016 - in pypy/branch/fast-forward: lib-python/modified-2.7.0/test pypy/objspace/std Message-ID: <20101213000306.C02C0282B90@codespeak.net> Author: afa Date: Mon Dec 13 01:03:05 2010 New Revision: 80016 Added: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_unpack.py - copied, changed from r79877, pypy/branch/fast-forward/lib-python/2.7.0/test/test_unpack.py Modified: pypy/branch/fast-forward/pypy/objspace/std/objspace.py Log: PyPy provides better error messages than CPython Copied: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_unpack.py (from r79877, pypy/branch/fast-forward/lib-python/2.7.0/test/test_unpack.py) ============================================================================== --- pypy/branch/fast-forward/lib-python/2.7.0/test/test_unpack.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_unpack.py Mon Dec 13 01:03:05 2010 @@ -62,14 +62,14 @@ >>> a, b = t Traceback (most recent call last): ... - ValueError: too many values to unpack + ValueError: expected length 2, got 3 Unpacking tuple of wrong size >>> a, b = l Traceback (most recent call last): ... - ValueError: too many values to unpack + ValueError: expected length 2, got 3 Unpacking sequence too short Modified: pypy/branch/fast-forward/pypy/objspace/std/objspace.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/objspace.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/objspace.py Mon Dec 13 01:03:05 2010 @@ -346,7 +346,7 @@ def _wrap_expected_length(self, expected, got): return OperationError(self.w_ValueError, - self.wrap("Expected length %d, got %d" % (expected, got))) + self.wrap("expected length %d, got %d" % (expected, got))) def unpackiterable(self, w_obj, expected_length=-1): if isinstance(w_obj, W_TupleObject): From agaynor at codespeak.net Mon Dec 13 06:45:24 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 13 Dec 2010 06:45:24 +0100 (CET) Subject: [pypy-svn] r80017 - in pypy/branch/fast-forward/pypy/module/_bisect: . test Message-ID: <20101213054524.8E765282BDC@codespeak.net> Author: agaynor Date: Mon Dec 13 06:45:22 2010 New Revision: 80017 Modified: pypy/branch/fast-forward/pypy/module/_bisect/interp_bisect.py pypy/branch/fast-forward/pypy/module/_bisect/test/test_bisect.py Log: Fixed an error in the bisect tests, verify that lo is non-negative. Modified: pypy/branch/fast-forward/pypy/module/_bisect/interp_bisect.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_bisect/interp_bisect.py (original) +++ pypy/branch/fast-forward/pypy/module/_bisect/interp_bisect.py Mon Dec 13 06:45:22 2010 @@ -1,3 +1,4 @@ +from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import ObjSpace, W_Root @@ -10,6 +11,10 @@ Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched.""" + if lo < 0: + raise OperationError(space.w_ValueError, + space.wrap("lo must be non-negative") + ) if hi == -1: hi = space.int_w(space.len(w_a)) while lo < hi: Modified: pypy/branch/fast-forward/pypy/module/_bisect/test/test_bisect.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/_bisect/test/test_bisect.py (original) +++ pypy/branch/fast-forward/pypy/module/_bisect/test/test_bisect.py Mon Dec 13 06:45:22 2010 @@ -42,6 +42,8 @@ assert bisect_left(a, 6, 0, 3) == 1 assert bisect_left(a, 6, 0, 4) == 1 + raises(ValueError, bisect_left, [1, 2, 3], 5, -1, 3) + def test_bisect_right(self): from _bisect import bisect_right a = [0, 5, 6, 6, 6, 7] From agaynor at codespeak.net Mon Dec 13 06:58:44 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 13 Dec 2010 06:58:44 +0100 (CET) Subject: [pypy-svn] r80018 - in pypy/branch/fast-forward/pypy: interpreter objspace/test Message-ID: <20101213055844.28945282BDC@codespeak.net> Author: agaynor Date: Mon Dec 13 06:58:42 2010 New Revision: 80018 Modified: pypy/branch/fast-forward/pypy/interpreter/function.py pypy/branch/fast-forward/pypy/objspace/test/test_descroperation.py Log: Delay verifying that classmethod is provided with a non-callable. Modified: pypy/branch/fast-forward/pypy/interpreter/function.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/function.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/function.py Mon Dec 13 06:58:42 2010 @@ -590,10 +590,6 @@ return space.wrap(Method(space, self.w_function, w_klass, space.w_None)) def descr_classmethod__new__(space, w_subtype, w_function): - if not space.is_true(space.callable(w_function)): - typename = space.type(w_function).getname(space, '?') - raise operationerrfmt(space.w_TypeError, - "'%s' object is not callable", typename) instance = space.allocate_instance(ClassMethod, w_subtype) instance.__init__(w_function) return space.wrap(instance) Modified: pypy/branch/fast-forward/pypy/objspace/test/test_descroperation.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/test/test_descroperation.py (original) +++ pypy/branch/fast-forward/pypy/objspace/test/test_descroperation.py Mon Dec 13 06:58:42 2010 @@ -493,6 +493,10 @@ raises(AttributeError, lambda: A().a) + def test_non_callable(self): + meth = classmethod(1).__get__(1) + raises(TypeError, meth) + def test_isinstance_and_issubclass(self): class Meta(type): def __instancecheck__(cls, instance): From agaynor at codespeak.net Mon Dec 13 07:07:27 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 13 Dec 2010 07:07:27 +0100 (CET) Subject: [pypy-svn] r80019 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20101213060727.7FC98282BDC@codespeak.net> Author: agaynor Date: Mon Dec 13 07:07:25 2010 New Revision: 80019 Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_typeobject.py pypy/branch/fast-forward/pypy/objspace/std/typeobject.py Log: A unicode string for __slots__ isn't considered as an iterable. Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_typeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_typeobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_typeobject.py Mon Dec 13 07:07:25 2010 @@ -664,6 +664,20 @@ assert c.a == 42 assert c.e == 85 + def test_string_slots(self): + class A(object): + __slots__ = "abc" + + class B(object): + __slots__ = u"abc" + + a = A() + a.abc = "awesome" + assert a.abc == "awesome" + b = B() + b.abc = "awesomer" + assert b.abc == "awesomer" + def test_base_attr(self): # check the '__base__' class A(object): @@ -1118,4 +1132,3 @@ return x + 1 a = A() assert a.f(1) == 2 - Modified: pypy/branch/fast-forward/pypy/objspace/std/typeobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/typeobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/typeobject.py Mon Dec 13 07:07:25 2010 @@ -556,7 +556,8 @@ wantdict = False wantweakref = False w_slots = dict_w['__slots__'] - if space.is_true(space.isinstance(w_slots, space.w_str)): + if (space.isinstance_w(w_slots, space.w_str) or + space.isinstance_w(w_slots, space.w_unicode)): slot_names_w = [w_slots] else: slot_names_w = space.unpackiterable(w_slots) From agaynor at codespeak.net Mon Dec 13 07:33:31 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Mon, 13 Dec 2010 07:33:31 +0100 (CET) Subject: [pypy-svn] r80020 - in pypy/branch/fast-forward/pypy/objspace/std: . test Message-ID: <20101213063331.5DC83282BDC@codespeak.net> Author: agaynor Date: Mon Dec 13 07:33:29 2010 New Revision: 80020 Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py Log: Make "\t\t".expandtabs(sys.maxint) raise an OverflowError with a useful error message. Modified: pypy/branch/fast-forward/pypy/objspace/std/stringobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/stringobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/stringobject.py Mon Dec 13 07:33:29 2010 @@ -671,12 +671,18 @@ def str_expandtabs__String_ANY(space, w_self, w_tabsize): u_self = w_self._value - u_tabsize = space.int_w(w_tabsize) + u_tabsize = space.int_w(w_tabsize) u_expanded = "" if u_self: split = u_self.split("\t") - u_expanded =oldtoken = split.pop(0) + try: + ovfcheck(len(split) * u_tabsize) + except OverflowError: + raise OperationError(space.w_OverflowError, + space.wrap("new string is too long") + ) + u_expanded = oldtoken = split.pop(0) for token in split: #print "%d#%d -%s-" % (_tabindent(oldtoken,u_tabsize), u_tabsize, token) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_stringobject.py Mon Dec 13 07:33:29 2010 @@ -351,6 +351,8 @@ raises(TypeError, 'hello'.endswith, (42,)) def test_expandtabs(self): + import sys + assert 'abc\rab\tdef\ng\thi'.expandtabs() == 'abc\rab def\ng hi' assert 'abc\rab\tdef\ng\thi'.expandtabs(8) == 'abc\rab def\ng hi' assert 'abc\rab\tdef\ng\thi'.expandtabs(4) == 'abc\rab def\ng hi' @@ -360,16 +362,18 @@ assert 'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == 'abc\r\nab\r\ndef\ng\r\nhi' s = 'xy\t' - assert s.expandtabs() =='xy ' + assert s.expandtabs() == 'xy ' s = '\txy\t' - assert s.expandtabs() ==' xy ' - assert s.expandtabs(1) ==' xy ' - assert s.expandtabs(2) ==' xy ' - assert s.expandtabs(3) ==' xy ' + assert s.expandtabs() == ' xy ' + assert s.expandtabs(1) == ' xy ' + assert s.expandtabs(2) == ' xy ' + assert s.expandtabs(3) == ' xy ' + + assert 'xy'.expandtabs() == 'xy' + assert ''.expandtabs() == '' - assert 'xy'.expandtabs() =='xy' - assert ''.expandtabs() =='' + raises(OverflowError, "t\tt\t".expandtabs, sys.maxint) def test_expandtabs_overflows_gracefully(self): import sys From afa at codespeak.net Mon Dec 13 08:26:22 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 13 Dec 2010 08:26:22 +0100 (CET) Subject: [pypy-svn] r80021 - pypy/branch/fast-forward/lib-python/modified-2.7.0/test Message-ID: <20101213072622.C4FBA282BDC@codespeak.net> Author: afa Date: Mon Dec 13 08:26:20 2010 New Revision: 80021 Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_optparse.py Log: Really apply decorator to the function :-( Modified: pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_optparse.py ============================================================================== --- pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_optparse.py (original) +++ pypy/branch/fast-forward/lib-python/modified-2.7.0/test/test_optparse.py Mon Dec 13 08:26:20 2010 @@ -383,7 +383,7 @@ self.assertRaises(self.parser.remove_option, ('foo',), None, ValueError, "no such option 'foo'") - test_support.impl_detail("sys.getrefcount") + @test_support.impl_detail("sys.getrefcount") def test_refleak(self): # If an OptionParser is carrying around a reference to a large # object, various cycles can prevent it from being GC'd in From afa at codespeak.net Mon Dec 13 08:51:28 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Mon, 13 Dec 2010 08:51:28 +0100 (CET) Subject: [pypy-svn] r80022 - in pypy/branch/fast-forward/pypy/module/itertools: . test Message-ID: <20101213075128.5A5A4282BDC@codespeak.net> Author: afa Date: Mon Dec 13 08:51:26 2010 New Revision: 80022 Modified: pypy/branch/fast-forward/pypy/module/itertools/interp_itertools.py pypy/branch/fast-forward/pypy/module/itertools/test/test_itertools.py Log: Fix an infinite loop with itertools.product(x, repeat=0) Modified: pypy/branch/fast-forward/pypy/module/itertools/interp_itertools.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/itertools/interp_itertools.py (original) +++ pypy/branch/fast-forward/pypy/module/itertools/interp_itertools.py Mon Dec 13 08:51:26 2010 @@ -1016,6 +1016,10 @@ self.cont = True def roll_gears(self): + if self.num_gears == 0: + self.cont = False + return + # Starting from the end of the gear indicies work to the front # incrementing the gear until the limit is reached. When the limit # is reached carry operation to the next gear Modified: pypy/branch/fast-forward/pypy/module/itertools/test/test_itertools.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/itertools/test/test_itertools.py (original) +++ pypy/branch/fast-forward/pypy/module/itertools/test/test_itertools.py Mon Dec 13 08:51:26 2010 @@ -752,3 +752,9 @@ l = [1, 2] m = ['a'] raises(TypeError, product, l, m, repeat=1, foo=2) + + def test_product_empty(self): + from itertools import product + prod = product('abc', repeat=0) + assert prod.next() == () + raises (StopIteration, prod.next) From commits-noreply at bitbucket.org Mon Dec 13 09:10:02 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 02:10:02 -0600 (CST) Subject: [pypy-svn] buildbot commit 924c923ef7ff: enable macosx64 tests nightly Message-ID: <20101213081002.32E126C140B@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User antocuni # Date 1291295463 0 # Node ID 924c923ef7ff3a0b7d9f37128a693cfb8ec02d0b # Parent fe16b21e1c0afbc2ee41493fd63c841d8e39a00d enable macosx64 tests nightly --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -199,6 +199,7 @@ BuildmasterConfig = { Nightly("nightly-0-45", [ JITBENCH, # on tannit -- nothing else there during first round! MACOSX32, # on minime + JITMACOSX64, # on mvt's machine ], hour=0, minute=45), Nightly("nightly-4-00", [ # rule: what we pick here on tannit should take at most 8 cores From commits-noreply at bitbucket.org Mon Dec 13 09:10:02 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 02:10:02 -0600 (CST) Subject: [pypy-svn] buildbot commit 1ea28ce0556a: schedule jitmacosx64 at 4:00 instead of 0:45 Message-ID: <20101213081002.3DA166C1417@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User antocuni # Date 1291297086 0 # Node ID 1ea28ce0556aa3996303a6e6fdbf206218263f45 # Parent 924c923ef7ff3a0b7d9f37128a693cfb8ec02d0b schedule jitmacosx64 at 4:00 instead of 0:45 --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -199,7 +199,6 @@ BuildmasterConfig = { Nightly("nightly-0-45", [ JITBENCH, # on tannit -- nothing else there during first round! MACOSX32, # on minime - JITMACOSX64, # on mvt's machine ], hour=0, minute=45), Nightly("nightly-4-00", [ # rule: what we pick here on tannit should take at most 8 cores @@ -210,6 +209,7 @@ BuildmasterConfig = { OJITLINUX32, # on tannit32, uses 1 core APPLVLWIN32, # on bigboard STACKLESSAPPLVLFREEBSD64, # on headless + JITMACOSX64, # on mvt's machine ], hour=4, minute=0), Nightly("nightly-6-00", [ # there should be only JITLINUX32 that takes a bit longer than From commits-noreply at bitbucket.org Mon Dec 13 09:10:01 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 02:10:01 -0600 (CST) Subject: [pypy-svn] buildbot commit 6f703848f295: Adapt the branch_order_prefixes. Message-ID: <20101213081001.F35046C12C2@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User arigo # Date 1289471336 0 # Node ID 6f703848f295d0300e6cd29165f3cfaa636ac585 # Parent e9d400615f5fb7957ae671885b1a7abf346beb0c Adapt the branch_order_prefixes. --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -86,12 +86,7 @@ status = WebStatus(httpPortNumber, allow # pypy test summary page summary = load('pypybuildbot.summary') -status.putChild('summary', summary.Summary(['own', 'applevel', - 'lib-python', 'jit', - 'stackless', - 'windows', 'mac', - 'benchmark-run', - 'other'])) +status.putChild('summary', summary.Summary(['linux', 'mac', 'win', 'freebsd'])) status.putChild('nightly', PyPyList(os.path.expanduser('~/nightly'), defaultType='application/octet-stream')) From commits-noreply at bitbucket.org Mon Dec 13 09:10:02 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 02:10:02 -0600 (CST) Subject: [pypy-svn] buildbot commit 13e8edc80748: Allow category prefixes too. Message-ID: <20101213081002.0D1196C12CC@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User arigo # Date 1289472039 0 # Node ID 13e8edc807488e727210b88900ad439b239040c6 # Parent 6f703848f295d0300e6cd29165f3cfaa636ac585 Allow category prefixes too. --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -86,7 +86,10 @@ status = WebStatus(httpPortNumber, allow # pypy test summary page summary = load('pypybuildbot.summary') -status.putChild('summary', summary.Summary(['linux', 'mac', 'win', 'freebsd'])) +status.putChild('summary', summary.Summary(categories=['linux', + 'mac', + 'win', + 'freebsd'])) status.putChild('nightly', PyPyList(os.path.expanduser('~/nightly'), defaultType='application/octet-stream')) --- a/bot2/pypybuildbot/summary.py +++ b/bot2/pypybuildbot/summary.py @@ -774,12 +774,12 @@ class Summary(HtmlResource): break else: branch_key = (len(self.branch_order_prefixes)+1, branch) - try: - i = self.categories.index(category) - cat_key = (0, i) - except ValueError: - cat_key = (1, category) - + for i, catprefix in enumerate(self.categories): + if category.startswith(catprefix): + break + else: + i = len(self.categories) + cat_key = (i, category) return cat_key + branch_key def body(self, request): From commits-noreply at bitbucket.org Mon Dec 13 09:10:01 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 02:10:01 -0600 (CST) Subject: [pypy-svn] buildbot commit e9d400615f5f: Tentative: change the category names to match Message-ID: <20101213081001.E8F976C12BB@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User arigo # Date 1289470877 0 # Node ID e9d400615f5fb7957ae671885b1a7abf346beb0c # Parent 3d998297bdc4880b009b6c2ac38cd4a0d592e90f Tentative: change the category names to match the platform. --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -226,91 +226,91 @@ BuildmasterConfig = { "slavenames": ["cobra", "bigdogvm1", "tannit32"], "builddir": LINUX32, "factory": pypyOwnTestFactory, - "category": 'own' + "category": 'linux32' }, {"name": LINUX64, "slavenames": ["tannit64"], "builddir": LINUX64, "factory": pypyOwnTestFactory, - "category": 'own64' + "category": 'linux64' }, {"name": MACOSX32, "slavenames": ["minime"], "builddir": MACOSX32, "factory": pypyOwnTestFactory, - "category": 'mac' + "category": 'mac32' }, {"name": WIN32, "slavenames": ["bigboard"], "builddir": WIN32, "factory": pypyOwnTestFactoryWin, - "category": 'own' + "category": 'win32' }, {"name": APPLVLLINUX32, "slavenames": ["bigdogvm1", "tannit32"], "builddir": APPLVLLINUX32, "factory": pypyTranslatedAppLevelTestFactory, - 'category': 'applevel' + 'category': 'linux32' }, {"name": APPLVLLINUX64, "slavenames": ["tannit64"], "builddir": APPLVLLINUX64, "factory": pypyTranslatedAppLevelTestFactory64, - "category": "applevel64" + "category": "linux64" }, {"name": STACKLESSAPPLVLLINUX32, "slavenames": ["bigdogvm1", "tannit32"], "builddir": STACKLESSAPPLVLLINUX32, "factory": pypyStacklessTranslatedAppLevelTestFactory, - "category": 'stackless' + "category": 'linux32-stackless' }, {"name": OJITLINUX32, "slavenames": ["bigdogvm1", "tannit32"], "builddir": OJITLINUX32, "factory": pypy_OjitTranslatedTestFactory, - "category": 'applevel' + "category": 'linux32' }, {"name": APPLVLWIN32, "slavenames": ["bigboard"], "builddir": APPLVLWIN32, "factory": pypyTranslatedAppLevelTestFactoryWin, - "category": "windows" + "category": "win32" }, {"name" : STACKLESSAPPLVLFREEBSD64, "slavenames": ['headless'], 'builddir' : STACKLESSAPPLVLFREEBSD64, 'factory' : pypyStacklessTranslatedAppLevelTestFactory, - "category": 'other' + "category": 'freebsd64-stackless' }, {"name" : JITLINUX32, "slavenames": ["bigdogvm1", "tannit32"], 'builddir' : JITLINUX32, 'factory' : pypyJITTranslatedTestFactory, - 'category' : 'jit', + 'category' : 'linux32', }, {'name': JITLINUX64, 'slavenames': ['tannit64'], 'builddir': JITLINUX64, 'factory': pypyJITTranslatedTestFactory64, - 'category': 'jit64', + 'category': 'linux64', }, {"name" : JITMACOSX64, "slavenames": ["macmini-mvt"], 'builddir' : JITMACOSX64, 'factory' : pypyJITTranslatedTestFactoryOSX, - 'category' : 'jit', + 'category' : 'mac64', }, {"name" : JITWIN32, "slavenames": ["bigboard"], 'builddir' : JITWIN32, 'factory' : pypyJITTranslatedTestFactoryWin, - 'category' : 'jit', + 'category' : 'win32', }, {"name": JITONLYLINUX32, "slavenames": ["tannit32", "bigdogvm1"], "builddir": JITONLYLINUX32, "factory": pypyJitOnlyOwnTestFactory, - "category": 'own' + "category": 'linux32' }, {"name": JITBENCH, "slavenames": ["tannit32"], From commits-noreply at bitbucket.org Mon Dec 13 09:10:01 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 02:10:01 -0600 (CST) Subject: [pypy-svn] buildbot commit 3d998297bdc4: add a builder for win32 own tests (without scheduling them nightly) Message-ID: <20101213081001.DCCB56C125F@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User antocuni # Date 1289382894 0 # Node ID 3d998297bdc4880b009b6c2ac38cd4a0d592e90f # Parent e4dfbfe4da0b34ae6dc4b394959e3f9f64a34eb9 add a builder for win32 own tests (without scheduling them nightly) --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -167,7 +167,7 @@ pypyJITBenchmarkFactory = pypybuilds.JIT LINUX32 = "own-linux-x86-32" LINUX64 = "own-linux-x86-64" MACOSX32 = "own-macosx-x86-32" -#WIN32 = "own-win-x86-32" +WIN32 = "own-win-x86-32" APPLVLLINUX32 = "pypy-c-app-level-linux-x86-32" APPLVLLINUX64 = "pypy-c-app-level-linux-x86-64" STACKLESSAPPLVLLINUX32 = "pypy-c-stackless-app-level-linux-x86-32" @@ -239,7 +239,13 @@ BuildmasterConfig = { "builddir": MACOSX32, "factory": pypyOwnTestFactory, "category": 'mac' - }, + }, + {"name": WIN32, + "slavenames": ["bigboard"], + "builddir": WIN32, + "factory": pypyOwnTestFactoryWin, + "category": 'own' + }, {"name": APPLVLLINUX32, "slavenames": ["bigdogvm1", "tannit32"], "builddir": APPLVLLINUX32, From commits-noreply at bitbucket.org Mon Dec 13 09:10:02 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 02:10:02 -0600 (CST) Subject: [pypy-svn] buildbot commit fe16b21e1c0a: Use the category name "osx64" explicitly. Message-ID: <20101213081002.273AA6C130E@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User arigo # Date 1290783837 0 # Node ID fe16b21e1c0afbc2ee41493fd63c841d8e39a00d # Parent 2797aa080f3489967f07b6f1d48a62d7bbdfe15b Use the category name "osx64" explicitly. --- a/bot2/pypybuildbot/master.py +++ b/bot2/pypybuildbot/master.py @@ -136,7 +136,16 @@ pypyJITTranslatedTestFactory64 = pypybui ) pypyJITTranslatedTestFactoryOSX = pypybuilds.Translated( - platform='osx', # works also for "OS/X 64" + platform='osx', + translationArgs=jit_translation_args, + targetArgs=[], + lib_python=True, + pypyjit=True, + app_tests=True, + ) + +pypyJITTranslatedTestFactoryOSX64 = pypybuilds.Translated( + platform='osx64', translationArgs=jit_translation_args, targetArgs=[], lib_python=True, @@ -295,7 +304,7 @@ BuildmasterConfig = { {"name" : JITMACOSX64, "slavenames": ["macmini-mvt"], 'builddir' : JITMACOSX64, - 'factory' : pypyJITTranslatedTestFactoryOSX, + 'factory' : pypyJITTranslatedTestFactoryOSX64, 'category' : 'mac64', }, {"name" : JITWIN32, From commits-noreply at bitbucket.org Mon Dec 13 09:10:02 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 02:10:02 -0600 (CST) Subject: [pypy-svn] buildbot commit 2797aa080f34: Properly escape URLs Message-ID: <20101213081002.1966E6C12DE@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User fijal # Date 1290409548 0 # Node ID 2797aa080f3489967f07b6f1d48a62d7bbdfe15b # Parent 13e8edc807488e727210b88900ad439b239040c6 Properly escape URLs --- a/bot2/pypybuildbot/pypylist.py +++ b/bot2/pypybuildbot/pypylist.py @@ -3,6 +3,7 @@ import datetime import itertools import re import py +import cgi from twisted.web import resource from twisted.web.static import File, DirectoryLister @@ -189,7 +190,7 @@ td,th {padding-left: 0.5em; padding-righ if branch == 'trunk': branch = '%3Ctrunk%3E' # if category: - href = '/summary?category=%s&branch=%s&recentrev=%s' % (category, branch, rev) + href = cgi.escape('/summary?category=%s&branch=%s&recentrev=%s' % (category, branch, rev)) str_summary = '%s' % (href, summary) else: str_summary = str(summary) From antocuni at codespeak.net Mon Dec 13 09:55:10 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 13 Dec 2010 09:55:10 +0100 (CET) Subject: [pypy-svn] r80023 - pypy/branch/jitypes2/lib_pypy/_ctypes Message-ID: <20101213085510.027ED282BDC@codespeak.net> Author: antocuni Date: Mon Dec 13 09:55:08 2010 New Revision: 80023 Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Log: actually pass the *value* of the pointer buffer (i.e., the address of the param which has been wrapped with byref). test_voidresult passes Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Mon Dec 13 09:55:08 2010 @@ -398,11 +398,7 @@ assert isinstance(value, basestring) and len(value) == 1 value = ord(value) elif argtype._ffishape == 'P': - value = arg._buffer.buffer - if value > sys.maxint: - # XXX: workaround for old versions of pypy-c, as soon as - # translation works again we can remove it - value = (-sys.maxint-1)*2 + value + value = arg._get_buffer_value() else: value = arg.value newargs.append(value) From cfbolz at codespeak.net Mon Dec 13 10:40:32 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 13 Dec 2010 10:40:32 +0100 (CET) Subject: [pypy-svn] r80024 - pypy/trunk/pypy/interpreter Message-ID: <20101213094032.31700282BDD@codespeak.net> Author: cfbolz Date: Mon Dec 13 10:40:26 2010 New Revision: 80024 Modified: pypy/trunk/pypy/interpreter/argument.py Log: fix test of r79907. apparently, no existing test needed this condition. Modified: pypy/trunk/pypy/interpreter/argument.py ============================================================================== --- pypy/trunk/pypy/interpreter/argument.py (original) +++ pypy/trunk/pypy/interpreter/argument.py Mon Dec 13 10:40:26 2010 @@ -103,9 +103,9 @@ make_sure_not_resized(self.keywords_w) make_sure_not_resized(self.arguments_w) - if w_stararg is not None and space.is_true(w_stararg): + if w_stararg is not None: self._combine_starargs_wrapped(w_stararg) - if w_starstararg is not None and space.is_true(w_starstararg): + if w_starstararg is not None: self._combine_starstarargs_wrapped(w_starstararg) # if we have a call where **args are used at the callsite # we shouldn't let the JIT see the argument matching From cfbolz at codespeak.net Mon Dec 13 10:47:00 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Mon, 13 Dec 2010 10:47:00 +0100 (CET) Subject: [pypy-svn] r80025 - in pypy/trunk/pypy/objspace/std: . test Message-ID: <20101213094700.CA62B282BDD@codespeak.net> Author: cfbolz Date: Mon Dec 13 10:46:59 2010 New Revision: 80025 Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py Log: implement an EmptyDictImplementation, that most dictionaries start out as. This makes dictionaries that are just allocated and never filled much more efficient (which apparently happens more often than you would think, judging by our memory studies). Modified: pypy/trunk/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/dictmultiobject.py Mon Dec 13 10:46:59 2010 @@ -55,9 +55,8 @@ else: if w_type is None: w_type = space.w_dict - w_self = space.allocate_instance(W_DictMultiObject, w_type) - W_DictMultiObject.__init__(w_self, space) - w_self.initialize_as_rdict() + w_self = space.allocate_instance(EmptyDictImplementation, w_type) + EmptyDictImplementation.__init__(w_self, space) return w_self def __init__(self, space): @@ -107,7 +106,7 @@ def impl_setitem_str(self, key, w_value): raise NotImplementedError("abstract base class") - def impl_setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): raise NotImplementedError("abstract base class") def impl_delitem(self, w_key): @@ -408,6 +407,45 @@ return self.shadowed[i] +class EmptyDictImplementation(W_DictMultiObject): + def __init__(self, space): + self.space = space + + def impl_setitem(self, w_key, w_value): + self._as_rdict().impl_fallback_setitem(w_key, w_value) + + def impl_setitem_str(self, key, w_value): + self._as_rdict().impl_fallback_setitem_str(key, w_value) + + def impl_delitem(self, w_key): + raise KeyError + + def impl_length(self): + return 0 + + def impl_getitem_str(self, key): + return None + + def impl_getitem(self, w_key): + # in case the key is unhashable, try to hash it + self.space.hash(w_key) + # return None anyway + return None + + def impl_iter(self): + # XXX I guess it's not important to be fast in this case? + return self._as_rdict().impl_fallback_iter() + + def impl_clear(self): + self.r_dict_content = None + + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() + return self + + def _clear_fields(self): + pass + class RDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) Modified: pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/trunk/pypy/objspace/std/test/test_dictmultiobject.py Mon Dec 13 10:46:59 2010 @@ -16,6 +16,7 @@ space = self.space d = self.space.newdict() assert not self.space.is_true(d) + assert d.r_dict_content is None def test_nonempty(self): space = self.space From antocuni at codespeak.net Mon Dec 13 11:02:46 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 13 Dec 2010 11:02:46 +0100 (CET) Subject: [pypy-svn] r80026 - pypy/branch/jitypes2/lib_pypy/_ctypes Message-ID: <20101213100246.1EE02282BDD@codespeak.net> Author: antocuni Date: Mon Dec 13 11:02:45 2010 New Revision: 80026 Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Log: support string arguments and results. Makes test_stringresult passing Modified: pypy/branch/jitypes2/lib_pypy/_ctypes/function.py ============================================================================== --- pypy/branch/jitypes2/lib_pypy/_ctypes/function.py (original) +++ pypy/branch/jitypes2/lib_pypy/_ctypes/function.py Mon Dec 13 11:02:45 2010 @@ -399,6 +399,8 @@ value = ord(value) elif argtype._ffishape == 'P': value = arg._get_buffer_value() + elif argtype._ffishape == 'z': + value = arg._get_buffer_value() else: value = arg.value newargs.append(value) @@ -419,6 +421,8 @@ result = None else: assert False, 'TODO' + elif restype._ffishape == 'z': + result = restype(result).value # XXX: maybe it's the general way to do it? return result def _build_result(self, restype, resbuffer, argtypes, argsandobjs): From fijal at codespeak.net Mon Dec 13 11:29:39 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 13 Dec 2010 11:29:39 +0100 (CET) Subject: [pypy-svn] r80027 - pypy/extradoc/sprintinfo/leysin-winter-2011 Message-ID: <20101213102939.305B5282BDD@codespeak.net> Author: fijal Date: Mon Dec 13 11:29:37 2010 New Revision: 80027 Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Log: I'm coming, hooray! Modified: pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt ============================================================================== --- pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt (original) +++ pypy/extradoc/sprintinfo/leysin-winter-2011/people.txt Mon Dec 13 11:29:37 2010 @@ -13,6 +13,7 @@ Armin Rigo --/23 private Antonio Cuni 15/22 ermina Michael Foord 16/22 (maybe) ermina +Maciej Fijalkowski 17/22 ermina ==================== ============== ======================= @@ -26,7 +27,6 @@ Anders Hammarquist ? ? Christian Tismer ? ? Niko Matsakis ? ? -Maciej Fijalkowski ? ? Toby Watson ? ? Paul deGrandis ? ? Michael Hudson ? ? From david at codespeak.net Mon Dec 13 11:56:23 2010 From: david at codespeak.net (david at codespeak.net) Date: Mon, 13 Dec 2010 11:56:23 +0100 (CET) Subject: [pypy-svn] r80028 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101213105623.83518282BE3@codespeak.net> Author: david Date: Mon Dec 13 11:56:20 2010 New Revision: 80028 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Log: Fix calls with parameters on the stack and guard_exception Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Mon Dec 13 11:56:20 2010 @@ -243,7 +243,6 @@ adr = args[0] cond = self._emit_call(adr, op.getarglist()[1:], regalloc, fcond, op.result, spill_all_regs=spill_all_regs) - descr = op.getdescr() #XXX Hack, Hack, Hack if op.result and not we_are_translated() and not isinstance(descr, LoopToken): @@ -282,9 +281,8 @@ n = stack_args*WORD self._adjust_sp(n, fcond=fcond) for i in range(4, n_args): - reg, box = regalloc._ensure_value_is_boxed(args[i], regalloc) - self.mc.STR_ri(reg.value, r.sp.value, (i-4)*WORD) - regalloc.possibly_free_var(box) + self.regalloc_mov(regalloc.loc(args[i]), r.ip) + self.mc.STR_ri(r.ip.value, r.sp.value, (i-4)*WORD) #the actual call self.mc.BL(adr) @@ -329,14 +327,14 @@ def emit_op_guard_exception(self, op, arglocs, regalloc, fcond): loc, loc1, resloc, pos_exc_value, pos_exception = arglocs[:5] failargs = arglocs[5:] - self.mc.LDR_ri(loc1.value, loc1.value) + self.mc.gen_load_int(loc1.value, pos_exception.value) + self.mc.LDR_ri(r.ip.value, loc1.value) - self.mc.CMP_rr(loc1.value, loc.value) + self.mc.CMP_rr(r.ip.value, loc.value) self._emit_guard(op, failargs, c.EQ, save_exc=True) - self.mc.gen_load_int(loc1.value, pos_exc_value.value, fcond) + self.mc.gen_load_int(loc.value, pos_exc_value.value, fcond) if resloc: - self.mc.LDR_ri(resloc.value, loc1.value) - self.mc.gen_load_int(loc.value, pos_exception.value, fcond) + self.mc.LDR_ri(resloc.value, loc.value) self.mc.MOV_ri(r.ip.value, 0) self.mc.STR_ri(r.ip.value, loc.value) self.mc.STR_ri(r.ip.value, loc1.value) Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Mon Dec 13 11:56:20 2010 @@ -304,15 +304,14 @@ arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) loc, box = self._ensure_value_is_boxed(arg0) boxes.append(box) - loc1, box = self._ensure_value_is_boxed( - ConstInt(self.assembler.cpu.pos_exception()), boxes) + box = TempBox() + loc1 = self.force_allocate_reg(box, boxes) boxes.append(box) if op.result in self.longevity: resloc = self.force_allocate_reg(op.result, boxes) boxes.append(resloc) else: resloc = None - # There is some redundancy here ?! pos_exc_value = imm(self.assembler.cpu.pos_exc_value()) pos_exception = imm(self.assembler.cpu.pos_exception()) arglocs = self._prepare_guard(op, [loc, loc1, resloc, pos_exc_value, pos_exception]) From arigo at codespeak.net Mon Dec 13 13:42:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 13:42:01 +0100 (CET) Subject: [pypy-svn] r80029 - in pypy/branch/jit-stackcheck/pypy: jit/backend jit/backend/llsupport jit/backend/x86 jit/backend/x86/test jit/metainterp rlib Message-ID: <20101213124201.4D98D50813@codespeak.net> Author: arigo Date: Mon Dec 13 13:41:58 2010 New Revision: 80029 Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/llsupport/llmodel.py pypy/branch/jit-stackcheck/pypy/jit/backend/model.py pypy/branch/jit-stackcheck/pypy/jit/backend/x86/assembler.py pypy/branch/jit-stackcheck/pypy/jit/backend/x86/regalloc.py pypy/branch/jit-stackcheck/pypy/jit/backend/x86/rx86.py pypy/branch/jit-stackcheck/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/jit-stackcheck/pypy/jit/metainterp/pyjitpl.py pypy/branch/jit-stackcheck/pypy/rlib/rstack.py Log: Implement stack checks at the beginning of the entry code for call_assembler. A test passes on 32-bit, still crashes on 64-bit... Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/backend/llsupport/llmodel.py Mon Dec 13 13:41:58 2010 @@ -115,6 +115,7 @@ self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = lambda: (0, 0, 0) def _setup_exception_handling_translated(self): @@ -138,9 +139,20 @@ # in the assignment to self.saved_exc_value, as needed. self.saved_exc_value = exc_value + from pypy.rlib import rstack + STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def insert_stack_check(): + startaddr = rstack._stack_get_start_adr() + length = rstack._stack_get_length() + f = llhelper(STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + slowpathaddr = rffi.cast(lltype.Signed, f) + return startaddr, length, slowpathaddr + self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = insert_stack_check def _setup_on_leave_jitted_untranslated(self): # assume we don't need a backend leave in this case Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/backend/model.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/backend/model.py Mon Dec 13 13:41:58 2010 @@ -8,6 +8,7 @@ done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + exit_frame_with_exception_v = -1 total_compiled_loops = 0 total_compiled_bridges = 0 total_freed_loops = 0 Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/backend/x86/assembler.py Mon Dec 13 13:41:58 2010 @@ -84,6 +84,7 @@ self.fail_boxes_count = 0 self._current_depths_cache = (0, 0) self.datablockwrapper = None + self.stack_check_slowpath_imm = imm0 self.teardown() def leave_jitted_hook(self): @@ -122,6 +123,7 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + self._build_stack_check_slowpath() debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') @@ -194,6 +196,82 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.malloc_fixedsize_slowpath2 = rawstart + _STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def _build_stack_check_slowpath(self): + from pypy.rlib import rstack + mc = codebuf.MachineCodeBlockWrapper() + mc.PUSH_r(ebp.value) + mc.MOV_rr(ebp.value, esp.value) + # + if IS_X86_64: + # on the x86_64, we have to save all the registers that may + # have been used to pass arguments + for reg in [edi, esi, edx, ecx, r8, r9]: + mc.PUSH_r(reg.value) + mc.SUB_ri(esp.value, 8*8) + for i in range(8): + mc.MOVSD_sx(8*i, i) # xmm0 to xmm7 + # + if IS_X86_32: + mc.LEA_rb(eax.value, +8) + mc.PUSH_r(eax.value) + elif IS_X86_64: + mc.LEA_rb(edi.value, +16) + mc.AND_ri(esp.value, -16) + # + f = llhelper(self._STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + addr = rffi.cast(lltype.Signed, f) + mc.CALL(imm(addr)) + # + mc.MOV(eax, heap(self.cpu.pos_exception())) + mc.TEST_rr(eax.value, eax.value) + mc.J_il8(rx86.Conditions['NZ'], 0) + jnz_location = mc.get_relative_pos() + # + if IS_X86_64: + # restore the registers + for i in range(7, -1, -1): + mc.MOVSD_xs(i, 8*i) + for i, reg in [(6, r9), (5, r8), (4, ecx), + (3, edx), (2, esi), (1, edi)]: + mc.MOV_rb(reg, -8*i) + # + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.RET() + # + # patch the JNZ above + offset = mc.get_relative_pos() - jnz_location + assert 0 < offset <= 127 + mc.overwrite(jnz_location-1, chr(offset)) + # clear the exception from the global position + mc.MOV(eax, heap(self.cpu.pos_exc_value())) + mc.MOV(heap(self.cpu.pos_exception()), imm0) + mc.MOV(heap(self.cpu.pos_exc_value()), imm0) + # save the current exception instance into fail_boxes_ptr[0] + adr = self.fail_boxes_ptr.get_addr_for_num(0) + mc.MOV(heap(adr), eax) + # call the helper function to set the GC flag on the fail_boxes_ptr + # array (note that there is no exception any more here) + addr = self.cpu.get_on_leave_jitted_int(save_exception=False) + mc.CALL(imm(addr)) + # + assert self.cpu.exit_frame_with_exception_v >= 0 + mc.MOV_ri(eax.value, self.cpu.exit_frame_with_exception_v) + # + # footer -- note the ADD, which skips the return address of this + # function, and will instead return to the caller's caller. Note + # also that we completely ignore the saved arguments, because we + # are interrupting the function. + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.ADD_ri(esp.value, WORD) + mc.RET() + # + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.stack_check_slowpath_imm = imm(rawstart) + def assemble_loop(self, inputargs, operations, looptoken, log): '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) @@ -468,6 +546,24 @@ for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) + def _call_header_with_stack_check(self): + startaddr, length, slowpathaddr = self.cpu.insert_stack_check() + if slowpathaddr == 0: + pass # no stack check (e.g. not translated) + else: + self.mc.MOV(eax, esp) # MOV eax, current + self.mc.SUB(eax, heap(startaddr)) # SUB eax, [startaddr] + self.mc.CMP(eax, imm(length)) # CMP eax, length + self.mc.J_il8(rx86.Conditions['B'], 0) # JB .skip + jb_location = self.mc.get_relative_pos() + self.mc.CALL(self.stack_check_slowpath_imm) # CALL slowpath + # patch the JB above # .skip: + offset = self.mc.get_relative_pos() - jb_location + assert 0 < offset <= 127 + self.mc.overwrite(jb_location-1, chr(offset)) + # + self._call_header() + def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -485,7 +581,7 @@ # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] @@ -526,7 +622,7 @@ unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/backend/x86/regalloc.py Mon Dec 13 13:41:58 2010 @@ -101,7 +101,6 @@ return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): - exc = False def __init__(self, assembler, translate_support_code=False): assert isinstance(translate_support_code, bool) @@ -428,7 +427,10 @@ locs = [self.loc(op.getarg(i)) for i in range(op.numargs())] locs_are_ref = [op.getarg(i).type == REF for i in range(op.numargs())] fail_index = self.assembler.cpu.get_fail_descr_number(op.getdescr()) - self.assembler.generate_failure(fail_index, locs, self.exc, + # note: no exception should currently be set in llop.get_exception_addr + # even if this finish may be an exit_frame_with_exception (in this case + # the exception instance is in locs[0]). + self.assembler.generate_failure(fail_index, locs, False, locs_are_ref) self.possibly_free_vars_for_op(op) Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/backend/x86/rx86.py Mon Dec 13 13:41:58 2010 @@ -347,6 +347,7 @@ INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') INSN_br = insn(rex_w, chr(base+1), register(2,8), stack_bp(1)) INSN_rb = insn(rex_w, chr(base+3), register(1,8), stack_bp(2)) + INSN_rj = insn(rex_w, chr(base+3), register(1,8), '\x05', immediate(2)) INSN_bi8 = insn(rex_w, '\x83', orbyte(base), stack_bp(1), immediate(2,'b')) INSN_bi32= insn(rex_w, '\x81', orbyte(base), stack_bp(1), immediate(2)) @@ -364,7 +365,7 @@ INSN_bi32(mc, offset, immed) INSN_bi._always_inline_ = True # try to constant-fold single_byte() - return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br + return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br, INSN_rj def select_8_or_32_bit_immed(insn_8, insn_32): def INSN(*args): @@ -442,12 +443,12 @@ # ------------------------------ Arithmetic ------------------------------ - ADD_ri, ADD_rr, ADD_rb, _, _ = common_modes(0) - OR_ri, OR_rr, OR_rb, _, _ = common_modes(1) - AND_ri, AND_rr, AND_rb, _, _ = common_modes(4) - SUB_ri, SUB_rr, SUB_rb, _, _ = common_modes(5) - XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) - CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) + ADD_ri, ADD_rr, ADD_rb, _, _, ADD_rj = common_modes(0) + OR_ri, OR_rr, OR_rb, _, _, OR_rj = common_modes(1) + AND_ri, AND_rr, AND_rb, _, _, AND_rj = common_modes(4) + SUB_ri, SUB_rr, SUB_rb, _, _, SUB_rj = common_modes(5) + XOR_ri, XOR_rr, XOR_rb, _, _, XOR_rj = common_modes(6) + CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br, CMP_rj = common_modes(7) CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) @@ -459,7 +460,6 @@ CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) - CMP_rj = insn(rex_w, '\x3B', register(1, 8), '\x05', immediate(2)) CMP32_mi = insn(rex_nw, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/backend/x86/test/test_ztranslation.py Mon Dec 13 13:41:58 2010 @@ -93,6 +93,9 @@ assert res == expected def test_direct_assembler_call_translates(self): + """Test CALL_ASSEMBLER and the recursion limit""" + from pypy.rlib.rstackovf import StackOverflow + class Thing(object): def __init__(self, val): self.val = val @@ -135,9 +138,35 @@ i += 1 return frame.thing.val - res = self.meta_interp(main, [0], inline=True, + driver2 = JitDriver(greens = [], reds = ['n']) + + def main2(bound): + try: + while portal2(bound) == -bound+1: + bound *= 2 + except StackOverflow: + pass + return bound + + def portal2(n): + while True: + driver2.jit_merge_point(n=n) + n -= 1 + if n <= 0: + return n + n = portal2(n) + assert portal2(10) == -9 + + def mainall(codeno, bound): + return main(codeno) + main2(bound) + + res = self.meta_interp(mainall, [0, 1], inline=True, policy=StopAtXPolicy(change)) - assert res == main(0) + print hex(res) + assert res & 255 == main(0) + bound = res & ~255 + assert 1024 <= bound <= 131072 + assert bound & (bound-1) == 0 # a power of two class TestTranslationRemoveTypePtrX86(CCompiledMixin): Modified: pypy/branch/jit-stackcheck/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/metainterp/pyjitpl.py Mon Dec 13 13:41:58 2010 @@ -1294,6 +1294,10 @@ num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) # + tokens = self.loop_tokens_exit_frame_with_exception_ref + num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) + self.cpu.exit_frame_with_exception_v = num + # self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): Modified: pypy/branch/jit-stackcheck/pypy/rlib/rstack.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/rlib/rstack.py (original) +++ pypy/branch/jit-stackcheck/pypy/rlib/rstack.py Mon Dec 13 13:41:58 2010 @@ -7,6 +7,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rarithmetic import r_uint +from pypy.rlib import rgc from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.lltypesystem.lloperation import llop @@ -67,13 +68,18 @@ return # # Else call the slow path + stack_check_slowpath(current) +stack_check._always_inline_ = True + + at rgc.no_collect +def stack_check_slowpath(current): if ord(_stack_too_big_slowpath(current)): - # # Now we are sure that the stack is really too big. Note that the # stack_unwind implementation is different depending on if stackless # is enabled. If it is it unwinds the stack, otherwise it simply # raises a RuntimeError. stack_unwind() +stack_check_slowpath._dont_inline_ = True # ____________________________________________________________ From arigo at codespeak.net Mon Dec 13 13:49:08 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 13:49:08 +0100 (CET) Subject: [pypy-svn] r80030 - in pypy/trunk/pypy: module/posix module/posix/test rlib rpython/module translator/c/test Message-ID: <20101213124908.5AB1A282BE9@codespeak.net> Author: arigo Date: Mon Dec 13 13:49:06 2010 New Revision: 80030 Modified: pypy/trunk/pypy/module/posix/__init__.py pypy/trunk/pypy/module/posix/interp_posix.py pypy/trunk/pypy/module/posix/test/test_posix2.py pypy/trunk/pypy/rlib/rposix.py pypy/trunk/pypy/rpython/module/ll_os.py pypy/trunk/pypy/translator/c/test/test_extfunc.py Log: Merge branch/more-posix, adding some more functions to the 'os' module. Modified: pypy/trunk/pypy/module/posix/__init__.py ============================================================================== --- pypy/trunk/pypy/module/posix/__init__.py (original) +++ pypy/trunk/pypy/module/posix/__init__.py Mon Dec 13 13:49:06 2010 @@ -71,6 +71,8 @@ if hasattr(os, 'chown'): interpleveldefs['chown'] = 'interp_posix.chown' + if hasattr(os, 'lchown'): + interpleveldefs['lchown'] = 'interp_posix.lchown' if hasattr(os, 'ftruncate'): interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' if hasattr(os, 'fsync'): @@ -86,6 +88,8 @@ if hasattr(os, 'kill') and sys.platform != 'win32': interpleveldefs['kill'] = 'interp_posix.kill' interpleveldefs['abort'] = 'interp_posix.abort' + if hasattr(os, 'killpg'): + interpleveldefs['killpg'] = 'interp_posix.killpg' if hasattr(os, 'getpid'): interpleveldefs['getpid'] = 'interp_posix.getpid' if hasattr(os, 'link'): @@ -115,6 +119,12 @@ interpleveldefs['ttyname'] = 'interp_posix.ttyname' if hasattr(os, 'getloadavg'): interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' + if hasattr(os, 'mkfifo'): + interpleveldefs['mkfifo'] = 'interp_posix.mkfifo' + if hasattr(os, 'mknod'): + interpleveldefs['mknod'] = 'interp_posix.mknod' + if hasattr(os, 'nice'): + interpleveldefs['nice'] = 'interp_posix.nice' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', Modified: pypy/trunk/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/trunk/pypy/module/posix/interp_posix.py (original) +++ pypy/trunk/pypy/module/posix/interp_posix.py Mon Dec 13 13:49:06 2010 @@ -153,6 +153,7 @@ ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] def fsync(space, w_fd): + """Force write of file with filedescriptor to disk.""" fd = space.c_filedescriptor_w(w_fd) try: os.fsync(fd) @@ -161,6 +162,8 @@ fsync.unwrap_spec = [ObjSpace, W_Root] def fdatasync(space, w_fd): + """Force write of file with filedescriptor to disk. +Does not force update of metadata.""" fd = space.c_filedescriptor_w(w_fd) try: os.fdatasync(fd) @@ -169,6 +172,8 @@ fdatasync.unwrap_spec = [ObjSpace, W_Root] def fchdir(space, w_fd): + """Change to the directory of the given file descriptor. fildes must be +opened on a directory, not a file.""" fd = space.c_filedescriptor_w(w_fd) try: os.fchdir(fd) @@ -549,6 +554,27 @@ raise wrap_oserror(space, e) rename.unwrap_spec = [ObjSpace, W_Root, W_Root] +def mkfifo(space, w_filename, mode=0666): + """Create a FIFO (a POSIX named pipe).""" + try: + dispatch_filename(rposix.mkfifo)(space, w_filename, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_filename) +mkfifo.unwrap_spec = [ObjSpace, W_Root, "c_int"] + +def mknod(space, w_filename, mode=0600, device=0): + """Create a filesystem node (file, device special file or named pipe) +named filename. mode specifies both the permissions to use and the +type of node to be created, being combined (bitwise OR) with one of +S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. For S_IFCHR and S_IFBLK, +device defines the newly created device special file (probably using +os.makedev()), otherwise it is ignored.""" + try: + dispatch_filename(rposix.mknod)(space, w_filename, mode, device) + except OSError, e: + raise wrap_oserror2(space, e, w_filename) +mknod.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] + def umask(space, mask): "Set the current numeric umask and return the previous umask." prevmask = os.umask(mask) @@ -572,6 +598,14 @@ raise wrap_oserror(space, e) kill.unwrap_spec = [ObjSpace, "c_int", "c_int"] +def killpg(space, pgid, sig): + "Kill a process group with a signal." + try: + os.killpg(pgid, sig) + except OSError, e: + raise wrap_oserror(space, e) +killpg.unwrap_spec = [ObjSpace, "c_int", "c_int"] + def abort(space): """Abort the interpreter immediately. This 'dumps core' or otherwise fails in the hardest way possible on the hosting operating system.""" @@ -974,6 +1008,14 @@ return space.w_None chown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] +def lchown(space, path, uid, gid): + try: + os.lchown(path, uid, gid) + except OSError, e: + raise wrap_oserror(space, e, path) + return space.w_None +lchown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] + def getloadavg(space): try: load = os.getloadavg() @@ -985,6 +1027,15 @@ space.wrap(load[2])]) getloadavg.unwrap_spec = [ObjSpace] +def nice(space, inc): + "Decrease the priority of process by inc and return the new priority." + try: + res = os.nice(inc) + except OSError, e: + raise wrap_oserror(space, e) + return space.wrap(res) +nice.unwrap_spec = [ObjSpace, "c_int"] + if _WIN: from pypy.rlib import rwin32 Modified: pypy/trunk/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/trunk/pypy/module/posix/test/test_posix2.py (original) +++ pypy/trunk/pypy/module/posix/test/test_posix2.py Mon Dec 13 13:49:06 2010 @@ -659,6 +659,67 @@ f.close() os.chown(self.path, os.getuid(), os.getgid()) + if hasattr(os, 'lchown'): + def test_lchown(self): + os = self.posix + os.unlink(self.path) + raises(OSError, os.lchown, self.path, os.getuid(), os.getgid()) + os.symlink('foobar', self.path) + os.lchown(self.path, os.getuid(), os.getgid()) + + if hasattr(os, 'mkfifo'): + def test_mkfifo(self): + os = self.posix + os.mkfifo(self.path2 + 'test_mkfifo', 0666) + st = os.lstat(self.path2 + 'test_mkfifo') + import stat + assert stat.S_ISFIFO(st.st_mode) + + if hasattr(os, 'mknod'): + def test_mknod(self): + import stat + os = self.posix + # not very useful: os.mknod() without specifying 'mode' + os.mknod(self.path2 + 'test_mknod-1') + st = os.lstat(self.path2 + 'test_mknod-1') + assert stat.S_ISREG(st.st_mode) + # os.mknod() with S_IFIFO + os.mknod(self.path2 + 'test_mknod-2', 0600 | stat.S_IFIFO) + st = os.lstat(self.path2 + 'test_mknod-2') + assert stat.S_ISFIFO(st.st_mode) + + def test_mknod_with_ifchr(self): + # os.mknod() with S_IFCHR + # -- usually requires root priviledges -- + os = self.posix + if hasattr(os.lstat('.'), 'st_rdev'): + import stat + try: + os.mknod(self.path2 + 'test_mknod-3', 0600 | stat.S_IFCHR, + 0x105) + except OSError, e: + skip("os.mknod() with S_IFCHR: got %r" % (e,)) + else: + st = os.lstat(self.path2 + 'test_mknod-3') + assert stat.S_ISCHR(st.st_mode) + assert st.st_rdev == 0x105 + + if hasattr(os, 'nice') and hasattr(os, 'fork') and hasattr(os, 'waitpid'): + def test_nice(self): + os = self.posix + myprio = os.nice(0) + # + pid = os.fork() + if pid == 0: # in the child + res = os.nice(3) + os._exit(res) + # + pid1, status1 = os.waitpid(pid, 0) + assert pid1 == pid + assert os.WIFEXITED(status1) + assert os.WEXITSTATUS(status1) == myprio + 3 + + class AppTestEnvironment(object): def setup_class(cls): cls.space = space Modified: pypy/trunk/pypy/rlib/rposix.py ============================================================================== --- pypy/trunk/pypy/rlib/rposix.py (original) +++ pypy/trunk/pypy/rlib/rposix.py Mon Dec 13 13:49:06 2010 @@ -135,6 +135,20 @@ else: return os.rmdir(path.as_bytes()) + at specialize.argtype(0) +def mkfifo(path, mode): + if isinstance(path, str): + os.mkfifo(path, mode) + else: + os.mkfifo(path.as_bytes(), mode) + + at specialize.argtype(0) +def mknod(path, mode, device): + if isinstance(path, str): + os.mknod(path, mode, device) + else: + os.mknod(path.as_bytes(), mode, device) + if os.name == 'nt': import nt def _getfullpathname(path): Modified: pypy/trunk/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/trunk/pypy/rpython/module/ll_os.py (original) +++ pypy/trunk/pypy/rpython/module/ll_os.py Mon Dec 13 13:49:06 2010 @@ -99,7 +99,7 @@ return 'll_os.ll_os_w' + name def registering_str_unicode(posixfunc, condition=True): - if not condition: + if not condition or posixfunc is None: return registering(None, condition=False) func_name = posixfunc.__name__ @@ -1129,6 +1129,19 @@ return extdef([str, int, int], None, "ll_os.ll_os_chown", llimpl=os_chown_llimpl) + @registering_if(os, 'lchown') + def register_os_lchown(self): + os_lchown = self.llexternal('lchown',[rffi.CCHARP, rffi.INT, rffi.INT], + rffi.INT) + + def os_lchown_llimpl(path, uid, gid): + res = os_lchown(path, uid, gid) + if res == -1: + raise OSError(rposix.get_errno(), "os_lchown failed") + + return extdef([str, int, int], None, "ll_os.ll_os_lchown", + llimpl=os_lchown_llimpl) + @registering_if(os, 'readlink') def register_os_readlink(self): os_readlink = self.llexternal('readlink', @@ -1323,6 +1336,33 @@ return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, export_name=traits.ll_os_name('rename')) + @registering_str_unicode(getattr(os, 'mkfifo', None)) + def register_os_mkfifo(self, traits): + os_mkfifo = self.llexternal(traits.posix_function_name('mkfifo'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) + + def mkfifo_llimpl(path, mode): + res = rffi.cast(lltype.Signed, os_mkfifo(path, mode)) + if res < 0: + raise OSError(rposix.get_errno(), "os_mkfifo failed") + + return extdef([traits.str, int], s_None, llimpl=mkfifo_llimpl, + export_name=traits.ll_os_name('mkfifo')) + + @registering_str_unicode(getattr(os, 'mknod', None)) + def register_os_mknod(self, traits): + os_mknod = self.llexternal(traits.posix_function_name('mknod'), + [traits.CCHARP, rffi.MODE_T, rffi.INT], + rffi.INT) # xxx: actually ^^^ dev_t + + def mknod_llimpl(path, mode, dev): + res = rffi.cast(lltype.Signed, os_mknod(path, mode, dev)) + if res < 0: + raise OSError(rposix.get_errno(), "os_mknod failed") + + return extdef([traits.str, int, int], s_None, llimpl=mknod_llimpl, + export_name=traits.ll_os_name('mknod')) + @registering(os.umask) def register_os_umask(self): os_umask = self.llexternal(underscore_on_windows+'umask', [rffi.MODE_T], rffi.MODE_T) @@ -1348,6 +1388,20 @@ return extdef([int, int], s_None, llimpl=kill_llimpl, export_name="ll_os.ll_os_kill") + @registering_if(os, 'killpg') + def register_os_killpg(self): + os_killpg = self.llexternal('killpg', [rffi.INT, rffi.INT], + rffi.INT) + + def killpg_llimpl(pid, sig): + res = rffi.cast(lltype.Signed, os_killpg(rffi.cast(rffi.INT, pid), + rffi.cast(rffi.INT, sig))) + if res < 0: + raise OSError(rposix.get_errno(), "os_killpg failed") + + return extdef([int, int], s_None, llimpl=killpg_llimpl, + export_name="ll_os.ll_os_killpg") + @registering_if(os, 'link') def register_os_link(self): os_link = self.llexternal('link', [rffi.CCHARP, rffi.CCHARP], @@ -1444,6 +1498,25 @@ return extdef([int], s_None, llimpl=_exit_llimpl, export_name="ll_os.ll_os__exit") + @registering_if(os, 'nice') + def register_os_nice(self): + os_nice = self.llexternal('nice', [rffi.INT], rffi.INT) + + def nice_llimpl(inc): + # Assume that the system provides a standard-compliant version + # of nice() that returns the new priority. Nowadays, FreeBSD + # might be the last major non-compliant system (xxx check me). + rposix.set_errno(0) + res = rffi.cast(lltype.Signed, os_nice(inc)) + if res == -1: + err = rposix.get_errno() + if err != 0: + raise OSError(err, "os_nice failed") + return res + + return extdef([int], int, llimpl=nice_llimpl, + export_name="ll_os.ll_os_nice") + # --------------------------- os.stat & variants --------------------------- @registering(os.fstat) Modified: pypy/trunk/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/trunk/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/trunk/pypy/translator/c/test/test_extfunc.py Mon Dec 13 13:49:06 2010 @@ -4,6 +4,7 @@ from pypy.tool.udir import udir from pypy.rlib.rarithmetic import r_longlong from pypy.translator.c.test.test_genc import compile +from pypy.translator.c.test.test_standalone import StandaloneTests posix = __import__(os.name) # note: clock synchronizes itself! @@ -404,6 +405,28 @@ assert os.path.exists(tmpfile2) assert not os.path.exists(tmpfile1) +if hasattr(os, 'mkfifo'): + def test_os_mkfifo(): + tmpfile = str(udir.join('test_os_mkfifo.txt')) + def does_stuff(): + os.mkfifo(tmpfile, 0666) + f1 = compile(does_stuff, []) + f1() + import stat + st = os.lstat(tmpfile) + assert stat.S_ISFIFO(st.st_mode) + +if hasattr(os, 'mknod'): + def test_os_mknod(): + import stat + tmpfile = str(udir.join('test_os_mknod.txt')) + def does_stuff(): + os.mknod(tmpfile, 0600 | stat.S_IFIFO, 0) + f1 = compile(does_stuff, []) + f1() + st = os.lstat(tmpfile) + assert stat.S_ISFIFO(st.st_mode) + def test_os_umask(): def does_stuff(): mask1 = os.umask(0660) @@ -516,6 +539,62 @@ # for what reason do they want us to shift by 8? See the doc assert status1 >> 8 == 4 +if hasattr(os, 'kill'): + def test_kill_to_send_sigusr1(): + import signal + from pypy.module.signal import interp_signal + def does_stuff(): + interp_signal.pypysig_setflag(signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR1) + interp_signal.pypysig_ignore(signal.SIGUSR1) + while True: + n = interp_signal.pypysig_poll() + if n < 0 or n == signal.SIGUSR1: + break + return n + f1 = compile(does_stuff, []) + got_signal = f1() + assert got_signal == signal.SIGUSR1 + +if hasattr(os, 'killpg'): + def test_killpg(): + import signal + from pypy.module.signal import interp_signal + def does_stuff(): + interp_signal.pypysig_setflag(signal.SIGUSR1) + os.killpg(os.getpgrp(), signal.SIGUSR1) + interp_signal.pypysig_ignore(signal.SIGUSR1) + while True: + n = interp_signal.pypysig_poll() + if n < 0 or n == signal.SIGUSR1: + break + return n + f1 = compile(does_stuff, []) + got_signal = f1() + assert got_signal == signal.SIGUSR1 + +if hasattr(os, 'chown') and hasattr(os, 'lchown'): + def test_os_chown_lchown(): + path1 = udir.join('test_os_chown_lchown-1.txt') + path2 = udir.join('test_os_chown_lchown-2.txt') + path1.write('foobar') + path2.mksymlinkto('some-broken-symlink') + tmpfile1 = str(path1) + tmpfile2 = str(path2) + def does_stuff(): + # xxx not really a test, just checks that they are callable + os.chown(tmpfile1, os.getuid(), os.getgid()) + os.lchown(tmpfile1, os.getuid(), os.getgid()) + os.lchown(tmpfile2, os.getuid(), os.getgid()) + try: + os.chown(tmpfile2, os.getuid(), os.getgid()) + except OSError: + pass + else: + raise AssertionError("os.chown(broken symlink) should raise") + f1 = compile(does_stuff, []) + f1() + # ____________________________________________________________ def _real_getenv(var): @@ -783,3 +862,19 @@ finally: os.chdir(localdir) assert res == True + +# ____________________________________________________________ + + +class TestExtFuncStandalone(StandaloneTests): + + if hasattr(os, 'nice'): + def test_os_nice(self): + def does_stuff(argv): + res = os.nice(3) + print 'os.nice returned', res + return 0 + t, cbuilder = self.compile(does_stuff) + data = cbuilder.cmdexec('') + res = os.nice(0) + 3 + assert data.startswith('os.nice returned %d\n' % res) From arigo at codespeak.net Mon Dec 13 13:49:17 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 13:49:17 +0100 (CET) Subject: [pypy-svn] r80031 - pypy/branch/more-posix Message-ID: <20101213124917.8C524282BEA@codespeak.net> Author: arigo Date: Mon Dec 13 13:49:16 2010 New Revision: 80031 Removed: pypy/branch/more-posix/ Log: Merged. From arigo at codespeak.net Mon Dec 13 13:50:15 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 13:50:15 +0100 (CET) Subject: [pypy-svn] r80032 - pypy/branch/getopt-appmain Message-ID: <20101213125015.7B65A282BE9@codespeak.net> Author: arigo Date: Mon Dec 13 13:50:14 2010 New Revision: 80032 Added: pypy/branch/getopt-appmain/ - copied from r80031, pypy/trunk/ Log: A branch in which to refactor the option parsing done by appmain.py, to be more compatible with CPython's. From arigo at codespeak.net Mon Dec 13 14:13:01 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 14:13:01 +0100 (CET) Subject: [pypy-svn] r80033 - pypy/branch/jit-stackcheck/pypy/jit/backend/x86 Message-ID: <20101213131301.3208C282BE9@codespeak.net> Author: arigo Date: Mon Dec 13 14:12:58 2010 New Revision: 80033 Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/x86/assembler.py pypy/branch/jit-stackcheck/pypy/jit/backend/x86/rx86.py Log: Fixes for 64-bit. Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/backend/x86/assembler.py Mon Dec 13 14:12:58 2010 @@ -235,7 +235,7 @@ mc.MOVSD_xs(i, 8*i) for i, reg in [(6, r9), (5, r8), (4, ecx), (3, edx), (2, esi), (1, edi)]: - mc.MOV_rb(reg, -8*i) + mc.MOV_rb(reg.value, -8*i) # mc.MOV_rr(esp.value, ebp.value) mc.POP_r(ebp.value) Modified: pypy/branch/jit-stackcheck/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/jit-stackcheck/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/jit-stackcheck/pypy/jit/backend/x86/rx86.py Mon Dec 13 14:12:58 2010 @@ -347,6 +347,7 @@ INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') INSN_br = insn(rex_w, chr(base+1), register(2,8), stack_bp(1)) INSN_rb = insn(rex_w, chr(base+3), register(1,8), stack_bp(2)) + INSN_rm = insn(rex_w, chr(base+3), register(1,8), mem_reg_plus_const(2)) INSN_rj = insn(rex_w, chr(base+3), register(1,8), '\x05', immediate(2)) INSN_bi8 = insn(rex_w, '\x83', orbyte(base), stack_bp(1), immediate(2,'b')) INSN_bi32= insn(rex_w, '\x81', orbyte(base), stack_bp(1), immediate(2)) @@ -365,7 +366,7 @@ INSN_bi32(mc, offset, immed) INSN_bi._always_inline_ = True # try to constant-fold single_byte() - return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br, INSN_rj + return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br, INSN_rm, INSN_rj def select_8_or_32_bit_immed(insn_8, insn_32): def INSN(*args): @@ -443,18 +444,17 @@ # ------------------------------ Arithmetic ------------------------------ - ADD_ri, ADD_rr, ADD_rb, _, _, ADD_rj = common_modes(0) - OR_ri, OR_rr, OR_rb, _, _, OR_rj = common_modes(1) - AND_ri, AND_rr, AND_rb, _, _, AND_rj = common_modes(4) - SUB_ri, SUB_rr, SUB_rb, _, _, SUB_rj = common_modes(5) - XOR_ri, XOR_rr, XOR_rb, _, _, XOR_rj = common_modes(6) - CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br, CMP_rj = common_modes(7) + ADD_ri, ADD_rr, ADD_rb, _, _, ADD_rm, ADD_rj = common_modes(0) + OR_ri, OR_rr, OR_rb, _, _, OR_rm, OR_rj = common_modes(1) + AND_ri, AND_rr, AND_rb, _, _, AND_rm, AND_rj = common_modes(4) + SUB_ri, SUB_rr, SUB_rb, _, _, SUB_rm, SUB_rj = common_modes(5) + XOR_ri, XOR_rr, XOR_rb, _, _, XOR_rm, XOR_rj = common_modes(6) + CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br, CMP_rm, CMP_rj = common_modes(7) CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32) - CMP_rm = insn(rex_w, '\x3B', register(1, 8), mem_reg_plus_const(2)) CMP_mr = insn(rex_w, '\x39', register(2, 8), mem_reg_plus_const(1)) CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) From arigo at codespeak.net Mon Dec 13 14:59:51 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 14:59:51 +0100 (CET) Subject: [pypy-svn] r80034 - in pypy/branch/fast-forward/pypy/translator/goal: . test2 Message-ID: <20101213135951.3E7DE282BE9@codespeak.net> Author: arigo Date: Mon Dec 13 14:59:48 2010 New Revision: 80034 Modified: pypy/branch/fast-forward/pypy/translator/goal/app_main.py pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py Log: With this particular (custom) logic, then we mimic more closely CPython's sys.flags.inspect and sys.flags.interactive. It also allows us to pass a previously skipped test. Modified: pypy/branch/fast-forward/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/goal/app_main.py (original) +++ pypy/branch/fast-forward/pypy/translator/goal/app_main.py Mon Dec 13 14:59:48 2010 @@ -360,6 +360,8 @@ if not sys.argv: # (relevant in case of "reload(sys)") sys.argv.append('') options["run_stdin"] = True + if not options["ignore_environment"] and os.getenv('PYTHONINSPECT'): + options["inspect"] = True if print_sys_flags: flag_opts = ["%s=%s" % (opt, int(value)) for opt, value in options.iteritems() @@ -373,6 +375,7 @@ return options def run_command_line(interactive, + inspect, run_command, no_site, run_module, @@ -427,15 +430,17 @@ signal.signal(signal.SIGXFSZ, signal.SIG_IGN) def inspect_requested(): - # We get an interactive prompt in one of the following two cases: + # We get an interactive prompt in one of the following three cases: # - # * insepct=True, either from the "-i" option or from the fact that - # we printed the banner; + # * interactive=True, from the "-i" option + # or + # * inspect=True and stdin is a tty # or # * PYTHONINSPECT is set and stdin is a tty. # return (interactive or - (readenv and os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) + ((inspect or (readenv and os.getenv('PYTHONINSPECT'))) + and sys.stdin.isatty())) success = True @@ -485,7 +490,7 @@ exec co_python_startup in mainmodule.__dict__ run_toplevel(run_it) # Then we need a prompt. - interactive = True + inspect = True else: # If not interactive, just read and execute stdin normally. def run_it(): Modified: pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/fast-forward/pypy/translator/goal/test2/test_app_main.py Mon Dec 13 14:59:48 2010 @@ -331,17 +331,16 @@ child.expect('>>> ') def test_clear_pythoninspect(self): - py.test.skip("obscure difference with CPython -- do we care?") - old = os.environ.get('PYTHONINSPECT', '') + os.environ['PYTHONINSPECT_'] = '1' try: path = getscript(""" import os del os.environ['PYTHONINSPECT'] """) child = self.spawn([path]) - xxx # do we expect a prompt or not? CPython gives one + child.expect('>>> ') finally: - os.environ['PYTHONINSPECT'] = old + del os.environ['PYTHONINSPECT_'] def test_stdout_flushes_before_stdin_blocks(self): # This doesn't really test app_main.py, but a behavior that From danchr at codespeak.net Mon Dec 13 16:17:00 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Mon, 13 Dec 2010 16:17:00 +0100 (CET) Subject: [pypy-svn] r80035 - pypy/trunk Message-ID: <20101213151700.3170F282BE9@codespeak.net> Author: danchr Date: Mon Dec 13 16:16:56 2010 New Revision: 80035 Added: pypy/trunk/.hgignore Log: Add .hgignore file. Added: pypy/trunk/.hgignore ============================================================================== --- (empty file) +++ pypy/trunk/.hgignore Mon Dec 13 16:16:56 2010 @@ -0,0 +1,53 @@ +syntax:glob +*.py[co] + +testresult +site-packages +pypy/module/cpyext/src/*.o +pypy/bin/pypy-c +pypy/translator/jvm/src/pypy/*.class +pypy/module/_stackless/test/ +pypy/module/cpyext/test/*.errors +pypy/doc/*.html +pypy/doc/basicblock.asc +pypy/doc/*.svninfo +pypy/translator/jvm/.project +pypy/translator/jvm/.classpath +pypy/translator/jvm/eclipse-bin +pypy/translator/benchmark/docutils +pypy/translator/benchmark/templess +pypy/translator/benchmark/gadfly +pypy/translator/benchmark/mako +pypy/translator/benchmark/bench-custom.benchmark_result +pypy/translator/benchmark/shootout_benchmarks +pypy/module/_stackless/ +pypy/translator/goal/pypy-translation-snapshot +pypy/translator/goal/pypy-c* +pypy/translator/goal/*.exe +pypy/translator/goal/target*-c +pypy/_cache +site-packages/*.egg +site-packages/*.pth +pypy/doc/statistic/*.html +pypy/doc/statistic/*.eps +pypy/doc/statistic/*.pdf +pypy/translator/cli/src/pypylib.dll +pypy/translator/cli/src/query.exe +pypy/translator/cli/src/main.exe +lib_pypy/ctypes_config_cache/_*_cache.py +lib_pypy/ctypes_config_cache/_*_*_.py +pypy/translator/cli/query-descriptions +pypy/doc/discussion/*.html +pypy/doc/discussion/ +include/*.h +include/*.inl +pypy/doc/config/*.html +pypy/doc/config/style.css +pypy/doc/config/ +pypy/doc/jit/*.html +pypy/doc/jit/style.css +pypy/doc/image/lattice1.png +pypy/doc/image/lattice2.png +pypy/doc/image/lattice3.png +pypy/doc/image/stackless_informal.png +pypy/doc/image/parsing_example*.png From danchr at codespeak.net Mon Dec 13 16:33:41 2010 From: danchr at codespeak.net (danchr at codespeak.net) Date: Mon, 13 Dec 2010 16:33:41 +0100 (CET) Subject: [pypy-svn] r80036 - pypy/trunk/pypy/module/sys Message-ID: <20101213153341.A1F63282BE9@codespeak.net> Author: danchr Date: Mon Dec 13 16:33:39 2010 New Revision: 80036 Modified: pypy/trunk/pypy/module/sys/__init__.py pypy/trunk/pypy/module/sys/version.py Log: Add sys._mercurial attribute. Based on the equivalent patch to CPython 2.7: Example output: $ ./python.exe -c 'import sys; print sys.subversion; print sys._mercurial' ('CPython', 'branches/release27-maint', '') ('CPython', 'add-hg-build-id.diff qbase qtip tip', 'a40affe1a0c3') $ ./pypy-c -c 'import sys; print(sys.subversion); print(sys._mercurial)' ('PyPy', '', '0') ('PyPy', 'default', 'e5edda84e0bb') The Mercurial version detection calls out to 'hg' rather than parsing the Mercurial dirstate directly. There are two reasons for this: 1) The Mercurial dirstate is a binary file, and its format is an implementation detail of Mercurial. 2) Tags are used in the output, and the Mercurial logic for determining what tags to apply is fairly complex and often extended by extensions. Modified: pypy/trunk/pypy/module/sys/__init__.py ============================================================================== --- pypy/trunk/pypy/module/sys/__init__.py (original) +++ pypy/trunk/pypy/module/sys/__init__.py Mon Dec 13 16:33:39 2010 @@ -59,6 +59,7 @@ 'pypy_version_info' : 'version.get_pypy_version_info(space)', 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', + '_mercurial' : 'version.get_mercurial_info(space)', 'hexversion' : 'version.get_hexversion(space)', 'displayhook' : 'hook.displayhook', Modified: pypy/trunk/pypy/module/sys/version.py ============================================================================== --- pypy/trunk/pypy/module/sys/version.py (original) +++ pypy/trunk/pypy/module/sys/version.py Mon Dec 13 16:33:39 2010 @@ -84,6 +84,39 @@ space.wrap(svnbranch), space.wrap(str(svn_revision()))]) +def get_mercurial_info(space): + '''Obtain Mercurial version information by invoking the 'hg' command.''' + # TODO: support extracting from .hg_archival.txt + import py + from subprocess import Popen, PIPE + + pypyroot = os.path.abspath(os.path.join(pypydir, '..')) + hgexe = py.path.local.sysfind('hg') + + if hgexe and os.path.isdir(os.path.join(pypyroot, '.hg')): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull + + p = Popen([str(hgexe), 'id', '-i', pypyroot], stdout=PIPE, env=env) + hgid = p.stdout.read().strip() + + p = Popen([str(hgexe), 'id', '-t', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip().split()[0] + + if hgtag == 'tip': + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip() + + return space.newtuple([space.wrap('PyPy'), + space.wrap(hgtag), + space.wrap(hgid)]) + else: + return space.w_None + def tuple2hex(ver): d = {'alpha': 0xA, 'beta': 0xB, From david at codespeak.net Mon Dec 13 16:35:29 2010 From: david at codespeak.net (david at codespeak.net) Date: Mon, 13 Dec 2010 16:35:29 +0100 (CET) Subject: [pypy-svn] r80037 - pypy/branch/arm-backend/pypy/jit/backend/arm Message-ID: <20101213153529.E4566282BE9@codespeak.net> Author: david Date: Mon Dec 13 16:35:28 2010 New Revision: 80037 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Log: Make backend translate again Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/assembler.py Mon Dec 13 16:35:28 2010 @@ -393,9 +393,9 @@ else: cb.gen_load_int(r.ip.value, n, cond=fcond) if rev: - cb.ADD_rr(r.sp.value, base_reg.value, reg.value, cond=fcond) + cb.ADD_rr(r.sp.value, base_reg.value, r.ip.value, cond=fcond) else: - cb.SUB_rr(r.sp.value, base_reg.value, reg.value, cond=fcond) + cb.SUB_rr(r.sp.value, base_reg.value, r.ip.value, cond=fcond) def _walk_operations(self, operations, regalloc): fcond=c.AL @@ -429,7 +429,9 @@ def assemble_bridge(self, faildescr, inputargs, operations): self.setup() self.debug = False - enc = rffi.cast(rffi.CCHARP, faildescr._failure_recovery_code) + code = faildescr._failure_recovery_code + assert isinstance(code, int) + enc = rffi.cast(rffi.CCHARP, code) longevity = compute_vars_longevity(inputargs, operations) regalloc = ARMRegisterManager(longevity, assembler=self, frame_manager=ARMFrameManager()) @@ -522,7 +524,7 @@ pass def make_operation_list(): - def notimplemented(self, op, regalloc, fcond): + def notimplemented(self, op, arglocs, regalloc, fcond): raise NotImplementedError, op operations = [None] * (rop._LAST+1) @@ -539,7 +541,7 @@ return operations def make_guard_operation_list(): - def notimplemented(self, op, guard_op, regalloc, fcond): + def notimplemented(self, op, guard_op, arglocs, regalloc, fcond): raise NotImplementedError, op guard_operations = [notimplemented] * rop._LAST for key, value in rop.__dict__.items(): Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/codebuilder.py Mon Dec 13 16:35:28 2010 @@ -121,12 +121,12 @@ return self._pos size_of_gen_load_int = 4 * WORD - #XXX use MOV_ri if value fits in imm + ofs_shift = zip(range(8, 25, 8), range(12, 0, -4)) def gen_load_int(self, r, value, cond=cond.AL): """r is the register number, value is the value to be loaded to the register""" self.MOV_ri(r, (value & 0xFF), cond=cond) - for offset, shift in zip(range(8, 25, 8), range(12, 0, -4)): + for offset, shift in self.ofs_shift: b = (value >> offset) & 0xFF if b == 0: continue Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/opassembler.py Mon Dec 13 16:35:28 2010 @@ -240,8 +240,9 @@ return fcond def emit_op_call(self, op, args, regalloc, fcond, spill_all_regs=False): - adr = args[0] - cond = self._emit_call(adr, op.getarglist()[1:], regalloc, fcond, + adr = args[0].value + arglist = op.getarglist()[1:] + cond = self._emit_call(adr, arglist, regalloc, fcond, op.result, spill_all_regs=spill_all_regs) descr = op.getdescr() #XXX Hack, Hack, Hack @@ -312,10 +313,6 @@ self.mc.MOV_rr(resloc.value, argloc.value) return fcond - def emit_op_cond_call_gc_wb(self, op, regalloc, fcond): - #XXX implement once gc support is in place - return fcond - def emit_op_guard_no_exception(self, op, arglocs, regalloc, fcond): loc = arglocs[0] failargs = arglocs[1:] @@ -340,9 +337,10 @@ self.mc.STR_ri(r.ip.value, loc1.value) return fcond - def emit_op_debug_merge_point(self, op, regalloc, fcond): + def emit_op_debug_merge_point(self, op, arglocs, regalloc, fcond): return fcond emit_op_jit_debug = emit_op_debug_merge_point + emit_op_cond_call_gc_wb = emit_op_debug_merge_point class FieldOpAssembler(object): @@ -753,22 +751,11 @@ result=result) def emit_op_new(self, op, arglocs, regalloc, fcond): - self._emit_call(self.malloc_func_addr, arglocs, - regalloc, result=op.result) - #XXX free args here, because _emit_call works on regalloc - regalloc.possibly_free_vars(arglocs) - regalloc.possibly_free_var(op.result) return fcond def emit_op_new_with_vtable(self, op, arglocs, regalloc, fcond): - classint = arglocs[-1].value - callargs = arglocs[:-1] - self._emit_call(self.malloc_func_addr, callargs, - regalloc, result=op.result) + classint = arglocs[0].value self.set_vtable(op.result, classint) - #XXX free args here, because _emit_call works on regalloc - regalloc.possibly_free_vars(callargs) - regalloc.possibly_free_var(op.result) return fcond def set_vtable(self, box, vtable): Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Mon Dec 13 16:35:28 2010 @@ -14,6 +14,7 @@ from pypy.jit.backend.llsupport import symbolic from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.jit.codewriter import heaptracker +from pypy.rlib.objectmodel import we_are_translated class TempInt(TempBox): type = INT @@ -86,7 +87,10 @@ del self.reg_bindings[var] self.free_regs.append(loc) except KeyError: - import pdb; pdb.set_trace() + if not we_are_translated(): + import pdb; pdb.set_trace() + else: + raise ValueError def _check_imm_arg(self, arg, size=0xFF, allow_zero=True): if isinstance(arg, ConstInt): @@ -244,7 +248,7 @@ prepare_op_int_invert = prepare_op_int_neg def prepare_op_call(self, op, fcond): - args = [rffi.cast(lltype.Signed, op.getarg(0).getint())] + args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))] return args def _prepare_guard(self, op, args=None): @@ -309,7 +313,7 @@ boxes.append(box) if op.result in self.longevity: resloc = self.force_allocate_reg(op.result, boxes) - boxes.append(resloc) + boxes.append(op.result) else: resloc = None pos_exc_value = imm(self.assembler.cpu.pos_exc_value()) @@ -555,17 +559,21 @@ def prepare_op_new(self, op, fcond): arglocs = self._prepare_args_for_new_op(op.getdescr()) - #XXX args are freed in assembler._emit_call - #self.possibly_free_vars(arglocs) + self.assembler._emit_call(self.assembler.malloc_func_addr, + arglocs, self, result=op.result) + self.possibly_free_vars(arglocs) self.possibly_free_var(op.result) - return arglocs + return [] def prepare_op_new_with_vtable(self, op, fcond): classint = op.getarg(0).getint() descrsize = heaptracker.vtable2descr(self.assembler.cpu, classint) - arglocs = self._prepare_args_for_new_op(descrsize) - arglocs.append(imm(classint)) - return arglocs + callargs = self._prepare_args_for_new_op(descrsize) + self.assembler._emit_call(self.assembler.malloc_func_addr, + callargs, self, result=op.result) + self.possibly_free_vars(callargs) + self.possibly_free_var(op.result) + return [imm(classint)] def prepare_op_new_array(self, op, fcond): gc_ll_descr = self.assembler.cpu.gc_ll_descr @@ -634,7 +642,7 @@ faildescr = guard_op.getdescr() fail_index = self.assembler.cpu.get_fail_descr_number(faildescr) self.assembler._write_fail_index(fail_index) - args = [rffi.cast(lltype.Signed, op.getarg(0).getint())] + args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))] # force all reg values to be spilled when calling self.assembler.emit_op_call(op, args, self, fcond, spill_all_regs=True) @@ -644,6 +652,7 @@ faildescr = guard_op.getdescr() fail_index = self.assembler.cpu.get_fail_descr_number(faildescr) self.assembler._write_fail_index(fail_index) + return [] def _prepare_args_for_new_op(self, new_args): gc_ll_descr = self.assembler.cpu.gc_ll_descr From arigo at codespeak.net Mon Dec 13 17:11:28 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 17:11:28 +0100 (CET) Subject: [pypy-svn] r80038 - in pypy/branch/getopt-appmain/pypy/translator/goal: . test2 Message-ID: <20101213161128.AFFB1282BE9@codespeak.net> Author: arigo Date: Mon Dec 13 17:11:26 2010 New Revision: 80038 Modified: pypy/branch/getopt-appmain/pypy/translator/goal/app_main.py pypy/branch/getopt-appmain/pypy/translator/goal/test2/test_app_main.py Log: Whack at app_main until I get a result that I'm kind of happy with. Add tests for parse_command_line(). Modified: pypy/branch/getopt-appmain/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/getopt-appmain/pypy/translator/goal/app_main.py (original) +++ pypy/branch/getopt-appmain/pypy/translator/goal/app_main.py Mon Dec 13 17:11:26 2010 @@ -22,6 +22,25 @@ originalexcepthook = sys.__excepthook__ +def handle_sys_exit(e): + # exit if we catch a w_SystemExit + exitcode = e.code + if exitcode is None: + exitcode = 0 + else: + try: + exitcode = int(exitcode) + except: + # not an integer: print it to stderr + try: + stderr = sys.stderr + except AttributeError: + pass # too bad + else: + print >> stderr, exitcode + exitcode = 1 + raise SystemExit(exitcode) + def run_toplevel(f, *fargs, **fkwds): """Calls f() and handles all OperationErrors. Intended use is to run the main program or one interactive statement. @@ -45,72 +64,58 @@ stdout.write('\n') except SystemExit, e: - # exit if we catch a w_SystemExit - exitcode = e.code - if exitcode is None: - exitcode = 0 - else: - try: - exitcode = int(exitcode) - except: - # not an integer: print it to stderr - try: - stderr = sys.stderr - except AttributeError: - pass # too bad - else: - print >> stderr, exitcode - exitcode = 1 - raise SystemExit(exitcode) + handle_sys_exit(e) + except: + display_exception() + return False + return True # success + +def display_exception(): + etype, evalue, etraceback = sys.exc_info() + try: + # extra debugging info in case the code below goes very wrong + if DEBUG and hasattr(sys, 'stderr'): + s = getattr(etype, '__name__', repr(etype)) + print >> sys.stderr, "debug: exception-type: ", s + print >> sys.stderr, "debug: exception-value:", str(evalue) + tbentry = etraceback + if tbentry: + while tbentry.tb_next: + tbentry = tbentry.tb_next + lineno = tbentry.tb_lineno + filename = tbentry.tb_frame.f_code.co_filename + print >> sys.stderr, "debug: exception-tb: %s:%d" % ( + filename, lineno) + + # set the sys.last_xxx attributes + sys.last_type = etype + sys.last_value = evalue + sys.last_traceback = etraceback + + # call sys.excepthook + hook = getattr(sys, 'excepthook', originalexcepthook) + hook(etype, evalue, etraceback) + return # done except: - etype, evalue, etraceback = sys.exc_info() try: - # extra debugging info in case the code below goes very wrong - if DEBUG and hasattr(sys, 'stderr'): - s = getattr(etype, '__name__', repr(etype)) - print >> sys.stderr, "debug: exception-type: ", s - print >> sys.stderr, "debug: exception-value:", str(evalue) - tbentry = etraceback - if tbentry: - while tbentry.tb_next: - tbentry = tbentry.tb_next - lineno = tbentry.tb_lineno - filename = tbentry.tb_frame.f_code.co_filename - print >> sys.stderr, "debug: exception-tb: %s:%d" % ( - filename, lineno) - - # set the sys.last_xxx attributes - sys.last_type = etype - sys.last_value = evalue - sys.last_traceback = etraceback - - # call sys.excepthook - hook = getattr(sys, 'excepthook', originalexcepthook) - hook(etype, evalue, etraceback) - return False # done - - except: - try: - stderr = sys.stderr - except AttributeError: - pass # too bad - else: - print >> stderr, 'Error calling sys.excepthook:' - originalexcepthook(*sys.exc_info()) - print >> stderr - print >> stderr, 'Original exception was:' + stderr = sys.stderr + except AttributeError: + pass # too bad + else: + print >> stderr, 'Error calling sys.excepthook:' + originalexcepthook(*sys.exc_info()) + print >> stderr + print >> stderr, 'Original exception was:' - # we only get here if sys.excepthook didn't do its job - originalexcepthook(etype, evalue, etraceback) - return False + # we only get here if sys.excepthook didn't do its job + originalexcepthook(etype, evalue, etraceback) - return True # success # ____________________________________________________________ # Option parsing -def print_info(): +def print_info(*args): try: options = sys.pypy_translation_info except AttributeError: @@ -120,15 +125,17 @@ optitems.sort() for name, value in optitems: print ' %51s: %s' % (name, value) + raise SystemExit -def print_help(): +def print_help(*args): print 'usage: %s [options]' % (sys.executable,) print __doc__.rstrip() if 'pypyjit' in sys.builtin_module_names: - print_jit_help() + _print_jit_help() print + raise SystemExit -def print_jit_help(): +def _print_jit_help(): import pypyjit items = pypyjit.defaults.items() items.sort() @@ -136,6 +143,18 @@ print ' --jit %s=N %slow-level JIT parameter (default %s)' % ( key, ' '*(18-len(key)), value) +def print_version(*args): + print "Python", sys.version + raise SystemExit + +def set_jit_option(options, jitparam, *args): + if 'pypyjit' not in sys.builtin_module_names: + print >> sys.stderr, ("Warning: No jit support in %s" % + (sys.executable,)) + else: + import pypyjit + pypyjit.set_param(jitparam) + class CommandLineError(Exception): pass @@ -171,18 +190,6 @@ else: IS_WINDOWS = False -def get_argument(option, argv, i): - arg = argv[i] - n = len(option) - if len(arg) > n: - return arg[n:], i - else: - i += 1 - if i >= len(argv): - raise CommandLineError('Argument expected for the %s option' % - option) - return argv[i], i - def get_library_path(executable): search = executable while 1: @@ -200,7 +207,7 @@ break # found! return newpath -def setup_initial_paths(executable, nanos, readenv=True, **extra): +def setup_initial_paths(executable, nanos, ignore_environment=False, **extra): # a substituted os if we are translated global os os = nanos @@ -221,6 +228,7 @@ sys.executable = os.path.abspath(executable) newpath = get_library_path(executable) + readenv = not ignore_environment path = readenv and os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath @@ -232,84 +240,181 @@ sys.path.append(dir) _seen[dir] = True +# Order is significant! +sys_flags = ( + "debug", + "py3k_warning", + "division_warning", + "division_new", + "inspect", + "interactive", + "optimize", + "dont_write_bytecode", + "no_user_site", + "no_site", + "ignore_environment", + "tabcheck", + "verbose", + "unicode", + "bytes_warning", +) + + +default_options = dict.fromkeys( + sys_flags + + ("run_command", + "run_module", + "run_stdin", + "warnoptions", + "unbuffered"), 0) + + +PYTHON26 = False + +def simple_option(options, name, iterargv): + options[name] += 1 + +def div_option(options, div, iterargv): + if div == "warn": + options["division_warning"] = 1 + elif div == "warnall": + options["division_warning"] = 2 + elif div == "new": + options["division_new"] = 1 + elif div != "old": + raise CommandLineError("invalid division option: %r" % (div,)) + +def c_option(options, runcmd, iterargv): + options["run_command"] = runcmd + return ['-c'] + list(iterargv) + +def m_option(options, runmodule, iterargv): + options["run_module"] = True + return [runmodule] + list(iterargv) + +def W_option(options, warnoption, iterargv): + options["warnoptions"].append(warnoption) + +def end_options(options, _, iterargv): + return list(iterargv) + +cmdline_options = { + # simple options just increment the counter of the options listed above + 'd': (simple_option, 'debug'), + 'i': (simple_option, 'interactive'), + 'O': (simple_option, 'optimize'), + 'S': (simple_option, 'no_site'), + 'E': (simple_option, 'ignore_environment'), + 't': (simple_option, 'tabcheck'), + 'v': (simple_option, 'verbose'), + 'U': (simple_option, 'unicode'), + 'u': (simple_option, 'unbuffered'), + # more complex options + 'Q': (div_option, Ellipsis), + 'c': (c_option, Ellipsis), + 'm': (m_option, Ellipsis), + 'W': (W_option, Ellipsis), + 'V': (print_version, None), + '--version': (print_version, None), + '--info': (print_info, None), + 'h': (print_help, None), + '--help': (print_help, None), + '--jit': (set_jit_option, Ellipsis), + '--': (end_options, None), + } + +if PYTHON26: + cmdline_options.update({ + '3': (simple_option, 'py3k_warning'), + 'B': (simple_option, 'dont_write_bytecode'), + 's': (simple_option, 'no_user_site'), + 'b': (simple_option, 'bytes_warning'), + }) + + +def handle_argument(c, options, iterargv, iterarg=iter(())): + function, funcarg = cmdline_options[c] + # + # If needed, fill in the real argument by taking it from the command line + if funcarg is Ellipsis: + remaining = list(iterarg) + if remaining: + funcarg = ''.join(remaining) + else: + try: + funcarg = iterargv.next() + except StopIteration: + if len(c) == 1: + c = '-' + c + raise CommandLineError('Argument expected for the %r option' % c) + # + return function(options, funcarg, iterargv) + def parse_command_line(argv): - go_interactive = False - run_command = False - import_site = True - i = 0 - run_module = False - run_stdin = False - warnoptions = [] - unbuffered = False - readenv = True - while i < len(argv): - arg = argv[i] - if not arg.startswith('-'): - break - if arg == '-i': - go_interactive = True - elif arg.startswith('-c'): - cmd, i = get_argument('-c', argv, i) - argv[i] = '-c' - run_command = True - break - elif arg == '-E': - readenv = False - elif arg == '-u': - unbuffered = True - elif arg == '-O' or arg == '-OO': - pass - elif arg == '--version' or arg == '-V': - print "Python", sys.version - return - elif arg == '--info': - print_info() - return - elif arg == '-h' or arg == '--help': - print_help() - return - elif arg == '-S': - import_site = False - elif arg == '-': - run_stdin = True - break # not an option but a file name representing stdin - elif arg.startswith('-m'): - module, i = get_argument('-m', argv, i) - argv[i] = module - run_module = True - break - elif arg.startswith('-W'): - warnoptions, i = get_argument('-W', argv, i) - elif arg.startswith('--jit'): - jitparam, i = get_argument('--jit', argv, i) - if 'pypyjit' not in sys.builtin_module_names: - print >> sys.stderr, ("Warning: No jit support in %s" % - (sys.executable,)) - else: - import pypyjit - pypyjit.set_param(jitparam) - elif arg == '--': - i += 1 - break # terminates option list + options = default_options.copy() + options['warnoptions'] = [] + # + iterargv = iter(argv) + argv = None + for arg in iterargv: + # + # If the next argument isn't at least two characters long or + # doesn't start with '-', stop processing + if len(arg) < 2 or arg[0] != '-': + if IS_WINDOWS and arg == '/?': # special case + print_help() + argv = [arg] + list(iterargv) # finishes processing + # + # If the next argument is directly in cmdline_options, handle + # it as a single argument + elif arg in cmdline_options: + argv = handle_argument(arg, options, iterargv) + # + # Else interpret the rest of the argument character by character else: - raise CommandLineError('unrecognized option %r' % (arg,)) - i += 1 - sys.argv[:] = argv[i:] # don't change the list that sys.argv is bound to - if not sys.argv: # (relevant in case of "reload(sys)") - sys.argv.append('') - run_stdin = True - return locals() + iterarg = iter(arg) + iterarg.next() # skip the '-' + for c in iterarg: + if c not in cmdline_options: + raise CommandLineError('Unknown option: -%s' % (c,)) + argv = handle_argument(c, options, iterargv, iterarg) + + if not argv: + argv = [''] + options["run_stdin"] = True + elif argv[0] == '-': + options["run_stdin"] = True + + # don't change the list that sys.argv is bound to + # (relevant in case of "reload(sys)") + sys.argv[:] = argv + + if (options["interactive"] or + (not options["ignore_environment"] and os.getenv('PYTHONINSPECT'))): + options["inspect"] = True + + if PYTHON26 and we_are_translated(): + flags = [options[flag] for flag in sys_flags] + sys.flags = type(sys.flags)(flags) + sys.py3kwarning = sys.flags.py3k_warning + +## if not we_are_translated(): +## for key in sorted(options): +## print '%40s: %s' % (key, options[key]) +## print '%40s: %s' % ("sys.argv", sys.argv) + + return options -def run_command_line(go_interactive, +def run_command_line(interactive, + inspect, run_command, - import_site, + no_site, run_module, run_stdin, warnoptions, unbuffered, - readenv, - cmd=None, + ignore_environment, **ignored): # with PyPy in top of CPython we can only have around 100 # but we need more in the translated PyPy for the compiler package @@ -325,14 +430,18 @@ mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule - if import_site: + if not no_site: try: import site except: print >> sys.stderr, "'import site' failed" + readenv = not ignore_environment + pythonwarnings = readenv and os.getenv('PYTHONWARNINGS') + if pythonwarnings: + warnoptions.extend(pythonwarnings.split(',')) if warnoptions: - sys.warnoptions.append(warnoptions) + sys.warnoptions[:] = warnoptions from warnings import _processoptions _processoptions(sys.warnoptions) @@ -352,26 +461,28 @@ signal.signal(signal.SIGXFSZ, signal.SIG_IGN) def inspect_requested(): - # We get an interactive prompt in one of the following two cases: + # We get an interactive prompt in one of the following three cases: # - # * go_interactive=True, either from the "-i" option or - # from the fact that we printed the banner; + # * interactive=True, from the "-i" option + # or + # * inspect=True and stdin is a tty # or # * PYTHONINSPECT is set and stdin is a tty. # - return (go_interactive or - (readenv and os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) + return (interactive or + ((inspect or (readenv and os.getenv('PYTHONINSPECT'))) + and sys.stdin.isatty())) success = True try: - if run_command: + if run_command != 0: # handle the "-c" command # Put '' on sys.path sys.path.insert(0, '') def run_it(): - exec cmd in mainmodule.__dict__ + exec run_command in mainmodule.__dict__ success = run_toplevel(run_it) elif run_module: # handle the "-m" command @@ -389,7 +500,7 @@ # put it's directory on sys.path sys.path.insert(0, '') - if go_interactive or sys.stdin.isatty(): + if interactive or sys.stdin.isatty(): # If stdin is a tty or if "-i" is specified, we print # a banner and run $PYTHONSTARTUP. print_banner() @@ -410,7 +521,7 @@ exec co_python_startup in mainmodule.__dict__ run_toplevel(run_it) # Then we need a prompt. - go_interactive = True + inspect = True else: # If not interactive, just read and execute stdin normally. def run_it(): @@ -426,14 +537,25 @@ sys.path.insert(0, scriptdir) success = run_toplevel(execfile, sys.argv[0], mainmodule.__dict__) - # start a prompt if requested + except SystemExit, e: + status = e.code if inspect_requested(): + display_exception() + else: + status = not success + + # start a prompt if requested + if inspect_requested(): + inteactive = False + try: from _pypy_interact import interactive_console success = run_toplevel(interactive_console, mainmodule) - except SystemExit, e: - return e.code - else: - return not success + except SystemExit, e: + status = e.code + else: + status = not success + + return status def resolvedirof(filename): try: @@ -461,8 +583,6 @@ except CommandLineError, e: print_error(str(e)) return 2 - if cmdline is None: - return 0 setup_initial_paths(executable, nanos, **cmdline) return run_command_line(**cmdline) Modified: pypy/branch/getopt-appmain/pypy/translator/goal/test2/test_app_main.py ============================================================================== --- pypy/branch/getopt-appmain/pypy/translator/goal/test2/test_app_main.py (original) +++ pypy/branch/getopt-appmain/pypy/translator/goal/test2/test_app_main.py Mon Dec 13 17:11:26 2010 @@ -43,6 +43,93 @@ """) +class TestParseCommandLine: + + def check_options(self, options, sys_argv, **expected): + assert sys.argv == sys_argv + for key, value in expected.items(): + assert options[key] == value + for key, value in options.items(): + if key not in expected: + assert not value, ( + "option %r has unexpectedly the value %r" % (key, value)) + + def check(self, argv, **expected): + import StringIO + from pypy.translator.goal import app_main + saved_sys_argv = sys.argv[:] + saved_sys_stdout = sys.stdout + saved_sys_stderr = sys.stdout + app_main.os = os + try: + sys.stdout = sys.stderr = StringIO.StringIO() + try: + options = app_main.parse_command_line(argv) + except SystemExit: + output = expected['output_contains'] + assert output in sys.stdout.getvalue() + else: + self.check_options(options, **expected) + finally: + sys.argv[:] = saved_sys_argv + sys.stdout = saved_sys_stdout + sys.stderr = saved_sys_stderr + + def test_all_combinations_I_can_think_of(self): + self.check([], sys_argv=[''], run_stdin=True) + self.check(['-'], sys_argv=['-'], run_stdin=True) + self.check(['-S'], sys_argv=[''], run_stdin=True, no_site=1) + self.check(['-OO'], sys_argv=[''], run_stdin=True, optimize=2) + self.check(['-O', '-O'], sys_argv=[''], run_stdin=True, optimize=2) + self.check(['-Qnew'], sys_argv=[''], run_stdin=True, division_new=1) + self.check(['-Qold'], sys_argv=[''], run_stdin=True, division_new=0) + self.check(['-Qwarn'], sys_argv=[''], run_stdin=True, division_warning=1) + self.check(['-Qwarnall'], sys_argv=[''], run_stdin=True, + division_warning=2) + self.check(['-Q', 'new'], sys_argv=[''], run_stdin=True, division_new=1) + self.check(['-SOQnew'], sys_argv=[''], run_stdin=True, + no_site=1, optimize=1, division_new=1) + self.check(['-SOQ', 'new'], sys_argv=[''], run_stdin=True, + no_site=1, optimize=1, division_new=1) + self.check(['-i'], sys_argv=[''], run_stdin=True, + interactive=1, inspect=1) + self.check(['-h'], output_contains='usage:') + self.check(['-S', '-tO', '-h'], output_contains='usage:') + self.check(['-S', '-thO'], output_contains='usage:') + self.check(['-S', '-tO', '--help'], output_contains='usage:') + self.check(['-S', '-tO', '--info'], output_contains='translation') + self.check(['-S', '-tO', '--version'], output_contains='Python') + self.check(['-S', '-tOV'], output_contains='Python') + self.check(['--jit', 'foobar', '-S'], sys_argv=[''], + run_stdin=True, no_site=1) + self.check(['-c', 'pass'], sys_argv=['-c'], run_command='pass') + self.check(['-cpass'], sys_argv=['-c'], run_command='pass') + self.check(['-cpass','x'], sys_argv=['-c','x'], run_command='pass') + self.check(['-Sc', 'pass'], sys_argv=['-c'], run_command='pass', + no_site=1) + self.check(['-Scpass'], sys_argv=['-c'], run_command='pass', no_site=1) + self.check(['-c', '', ''], sys_argv=['-c', ''], run_command='') + self.check(['-mfoo', 'bar', 'baz'], sys_argv=['foo', 'bar', 'baz'], + run_module=True) + self.check(['-m', 'foo', 'bar', 'baz'], sys_argv=['foo', 'bar', 'baz'], + run_module=True) + self.check(['-Smfoo', 'bar', 'baz'], sys_argv=['foo', 'bar', 'baz'], + run_module=True, no_site=1) + self.check(['-Sm', 'foo', 'bar', 'baz'], sys_argv=['foo', 'bar', 'baz'], + run_module=True, no_site=1) + self.check(['-', 'foo', 'bar'], sys_argv=['-', 'foo', 'bar'], + run_stdin=True) + self.check(['foo', 'bar'], sys_argv=['foo', 'bar']) + self.check(['foo', '-i'], sys_argv=['foo', '-i']) + self.check(['-i', 'foo'], sys_argv=['foo'], interactive=1, inspect=1) + self.check(['--', 'foo'], sys_argv=['foo']) + self.check(['--', '-i', 'foo'], sys_argv=['-i', 'foo']) + self.check(['--', '-', 'foo'], sys_argv=['-', 'foo'], run_stdin=True) + self.check(['-Wbog'], sys_argv=[''], warnoptions=['bog'], run_stdin=True) + self.check(['-W', 'ab', '-SWc'], sys_argv=[''], warnoptions=['ab', 'c'], + run_stdin=True, no_site=1) + + class TestInteraction: """ These tests require pexpect (UNIX-only). @@ -416,7 +503,7 @@ def test_option_W_crashing(self): data = self.run('-W') - assert 'Argument expected for the -W option' in data + assert "Argument expected for the '-W' option" in data def test_option_W_arg_ignored(self): data = self.run('-Wc') From commits-noreply at bitbucket.org Mon Dec 13 17:11:49 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 10:11:49 -0600 (CST) Subject: [pypy-svn] pypy commit 85b74c8de1c9: import svn externals Message-ID: <20101213161149.1910D1E128B@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292256528 -3600 # Node ID 85b74c8de1c96027c19741ddac3fb967390d7235 # Parent e1895c78a34fa0f7d2e3ea8add913aa4e32a70e0 # Parent bf2c629d00711b091509f687151a057cf3098b13 import svn externals From commits-noreply at bitbucket.org Mon Dec 13 17:11:48 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 10:11:48 -0600 (CST) Subject: [pypy-svn] pypy commit bf2c629d0071: import svn externals needed for pypy (at codespeak svn revision 80021) Message-ID: <20101213161148.760721E1071@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292256361 -3600 # Node ID bf2c629d00711b091509f687151a057cf3098b13 import svn externals needed for pypy (at codespeak svn revision 80021) --- /dev/null +++ b/.hgsub @@ -0,0 +1,4 @@ +greenlet = [svn]http://codespeak.net/svn/greenlet/trunk/c +testrunner = [svn]http://codespeak.net/svn/pypy/build/testrunner +lib_pypy/pyrepl = [svn]http://codespeak.net/svn/pyrepl/trunk/pyrepl/pyrepl +lib_pypy/sqlite3 = [svn]http://codespeak.net/svn/pypy/pysqlite2 --- /dev/null +++ b/.hgsubstate @@ -0,0 +1,4 @@ +80037 greenlet +80037 lib_pypy/pyrepl +80037 lib_pypy/sqlite3 +80037 testrunner From commits-noreply at bitbucket.org Mon Dec 13 17:26:36 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 10:26:36 -0600 (CST) Subject: [pypy-svn] pypy commit 6f6dc96c294b: import svn externals in this branch too Message-ID: <20101213162636.B73B32403CD@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292257544 -3600 # Node ID 6f6dc96c294b8140ad2cffc17c0f2eb185fe4be5 # Parent 1414d7583324385691dbcbe1215bf8fe839d756f # Parent bf2c629d00711b091509f687151a057cf3098b13 import svn externals in this branch too From commits-noreply at bitbucket.org Mon Dec 13 17:58:21 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 10:58:21 -0600 (CST) Subject: [pypy-svn] pypy commit c1dae116dd3e: Make backend translate again Message-ID: <20101213165821.092C41E12DA@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User David Schneider # Date 1292254528 0 # Node ID c1dae116dd3ea2d26b66e5324f99029203113ec2 # Parent 4268e926c86c96a08c61a75c483a5fdf71d15e9a Make backend translate again --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -393,9 +393,9 @@ class AssemblerARM(ResOpAssembler): else: cb.gen_load_int(r.ip.value, n, cond=fcond) if rev: - cb.ADD_rr(r.sp.value, base_reg.value, reg.value, cond=fcond) + cb.ADD_rr(r.sp.value, base_reg.value, r.ip.value, cond=fcond) else: - cb.SUB_rr(r.sp.value, base_reg.value, reg.value, cond=fcond) + cb.SUB_rr(r.sp.value, base_reg.value, r.ip.value, cond=fcond) def _walk_operations(self, operations, regalloc): fcond=c.AL @@ -429,7 +429,9 @@ class AssemblerARM(ResOpAssembler): def assemble_bridge(self, faildescr, inputargs, operations): self.setup() self.debug = False - enc = rffi.cast(rffi.CCHARP, faildescr._failure_recovery_code) + code = faildescr._failure_recovery_code + assert isinstance(code, int) + enc = rffi.cast(rffi.CCHARP, code) longevity = compute_vars_longevity(inputargs, operations) regalloc = ARMRegisterManager(longevity, assembler=self, frame_manager=ARMFrameManager()) @@ -522,7 +524,7 @@ class AssemblerARM(ResOpAssembler): pass def make_operation_list(): - def notimplemented(self, op, regalloc, fcond): + def notimplemented(self, op, arglocs, regalloc, fcond): raise NotImplementedError, op operations = [None] * (rop._LAST+1) @@ -539,7 +541,7 @@ def make_operation_list(): return operations def make_guard_operation_list(): - def notimplemented(self, op, guard_op, regalloc, fcond): + def notimplemented(self, op, guard_op, arglocs, regalloc, fcond): raise NotImplementedError, op guard_operations = [notimplemented] * rop._LAST for key, value in rop.__dict__.items(): --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -240,8 +240,9 @@ class OpAssembler(object): return fcond def emit_op_call(self, op, args, regalloc, fcond, spill_all_regs=False): - adr = args[0] - cond = self._emit_call(adr, op.getarglist()[1:], regalloc, fcond, + adr = args[0].value + arglist = op.getarglist()[1:] + cond = self._emit_call(adr, arglist, regalloc, fcond, op.result, spill_all_regs=spill_all_regs) descr = op.getdescr() #XXX Hack, Hack, Hack @@ -312,10 +313,6 @@ class OpAssembler(object): self.mc.MOV_rr(resloc.value, argloc.value) return fcond - def emit_op_cond_call_gc_wb(self, op, regalloc, fcond): - #XXX implement once gc support is in place - return fcond - def emit_op_guard_no_exception(self, op, arglocs, regalloc, fcond): loc = arglocs[0] failargs = arglocs[1:] @@ -340,9 +337,10 @@ class OpAssembler(object): self.mc.STR_ri(r.ip.value, loc1.value) return fcond - def emit_op_debug_merge_point(self, op, regalloc, fcond): + def emit_op_debug_merge_point(self, op, arglocs, regalloc, fcond): return fcond emit_op_jit_debug = emit_op_debug_merge_point + emit_op_cond_call_gc_wb = emit_op_debug_merge_point class FieldOpAssembler(object): @@ -753,22 +751,11 @@ class AllocOpAssembler(object): result=result) def emit_op_new(self, op, arglocs, regalloc, fcond): - self._emit_call(self.malloc_func_addr, arglocs, - regalloc, result=op.result) - #XXX free args here, because _emit_call works on regalloc - regalloc.possibly_free_vars(arglocs) - regalloc.possibly_free_var(op.result) return fcond def emit_op_new_with_vtable(self, op, arglocs, regalloc, fcond): - classint = arglocs[-1].value - callargs = arglocs[:-1] - self._emit_call(self.malloc_func_addr, callargs, - regalloc, result=op.result) + classint = arglocs[0].value self.set_vtable(op.result, classint) - #XXX free args here, because _emit_call works on regalloc - regalloc.possibly_free_vars(callargs) - regalloc.possibly_free_var(op.result) return fcond def set_vtable(self, box, vtable): --- a/pypy/jit/backend/arm/codebuilder.py +++ b/pypy/jit/backend/arm/codebuilder.py @@ -121,12 +121,12 @@ class AbstractARMv7Builder(object): return self._pos size_of_gen_load_int = 4 * WORD - #XXX use MOV_ri if value fits in imm + ofs_shift = zip(range(8, 25, 8), range(12, 0, -4)) def gen_load_int(self, r, value, cond=cond.AL): """r is the register number, value is the value to be loaded to the register""" self.MOV_ri(r, (value & 0xFF), cond=cond) - for offset, shift in zip(range(8, 25, 8), range(12, 0, -4)): + for offset, shift in self.ofs_shift: b = (value >> offset) & 0xFF if b == 0: continue --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -14,6 +14,7 @@ from pypy.jit.backend.llsupport.descr im from pypy.jit.backend.llsupport import symbolic from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.jit.codewriter import heaptracker +from pypy.rlib.objectmodel import we_are_translated class TempInt(TempBox): type = INT @@ -86,7 +87,10 @@ class ARMRegisterManager(RegisterManager del self.reg_bindings[var] self.free_regs.append(loc) except KeyError: - import pdb; pdb.set_trace() + if not we_are_translated(): + import pdb; pdb.set_trace() + else: + raise ValueError def _check_imm_arg(self, arg, size=0xFF, allow_zero=True): if isinstance(arg, ConstInt): @@ -244,7 +248,7 @@ class ARMRegisterManager(RegisterManager prepare_op_int_invert = prepare_op_int_neg def prepare_op_call(self, op, fcond): - args = [rffi.cast(lltype.Signed, op.getarg(0).getint())] + args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))] return args def _prepare_guard(self, op, args=None): @@ -309,7 +313,7 @@ class ARMRegisterManager(RegisterManager boxes.append(box) if op.result in self.longevity: resloc = self.force_allocate_reg(op.result, boxes) - boxes.append(resloc) + boxes.append(op.result) else: resloc = None pos_exc_value = imm(self.assembler.cpu.pos_exc_value()) @@ -555,17 +559,21 @@ class ARMRegisterManager(RegisterManager def prepare_op_new(self, op, fcond): arglocs = self._prepare_args_for_new_op(op.getdescr()) - #XXX args are freed in assembler._emit_call - #self.possibly_free_vars(arglocs) + self.assembler._emit_call(self.assembler.malloc_func_addr, + arglocs, self, result=op.result) + self.possibly_free_vars(arglocs) self.possibly_free_var(op.result) - return arglocs + return [] def prepare_op_new_with_vtable(self, op, fcond): classint = op.getarg(0).getint() descrsize = heaptracker.vtable2descr(self.assembler.cpu, classint) - arglocs = self._prepare_args_for_new_op(descrsize) - arglocs.append(imm(classint)) - return arglocs + callargs = self._prepare_args_for_new_op(descrsize) + self.assembler._emit_call(self.assembler.malloc_func_addr, + callargs, self, result=op.result) + self.possibly_free_vars(callargs) + self.possibly_free_var(op.result) + return [imm(classint)] def prepare_op_new_array(self, op, fcond): gc_ll_descr = self.assembler.cpu.gc_ll_descr @@ -634,7 +642,7 @@ class ARMRegisterManager(RegisterManager faildescr = guard_op.getdescr() fail_index = self.assembler.cpu.get_fail_descr_number(faildescr) self.assembler._write_fail_index(fail_index) - args = [rffi.cast(lltype.Signed, op.getarg(0).getint())] + args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))] # force all reg values to be spilled when calling self.assembler.emit_op_call(op, args, self, fcond, spill_all_regs=True) @@ -644,6 +652,7 @@ class ARMRegisterManager(RegisterManager faildescr = guard_op.getdescr() fail_index = self.assembler.cpu.get_fail_descr_number(faildescr) self.assembler._write_fail_index(fail_index) + return [] def _prepare_args_for_new_op(self, new_args): gc_ll_descr = self.assembler.cpu.gc_ll_descr From commits-noreply at bitbucket.org Mon Dec 13 17:58:21 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 10:58:21 -0600 (CST) Subject: [pypy-svn] pypy commit 805815216ca7: Add .hgignore file. Message-ID: <20101213165821.9B57A1E14B5@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Dan Villiom Podlaski Christiansen # Date 1292253416 0 # Node ID 805815216ca7038c2ac1bc6cd620b28b96aef298 # Parent e1895c78a34fa0f7d2e3ea8add913aa4e32a70e0 Add .hgignore file. --- /dev/null +++ b/.hgignore @@ -0,0 +1,53 @@ +syntax:glob +*.py[co] + +testresult +site-packages +pypy/module/cpyext/src/*.o +pypy/bin/pypy-c +pypy/translator/jvm/src/pypy/*.class +pypy/module/_stackless/test/ +pypy/module/cpyext/test/*.errors +pypy/doc/*.html +pypy/doc/basicblock.asc +pypy/doc/*.svninfo +pypy/translator/jvm/.project +pypy/translator/jvm/.classpath +pypy/translator/jvm/eclipse-bin +pypy/translator/benchmark/docutils +pypy/translator/benchmark/templess +pypy/translator/benchmark/gadfly +pypy/translator/benchmark/mako +pypy/translator/benchmark/bench-custom.benchmark_result +pypy/translator/benchmark/shootout_benchmarks +pypy/module/_stackless/ +pypy/translator/goal/pypy-translation-snapshot +pypy/translator/goal/pypy-c* +pypy/translator/goal/*.exe +pypy/translator/goal/target*-c +pypy/_cache +site-packages/*.egg +site-packages/*.pth +pypy/doc/statistic/*.html +pypy/doc/statistic/*.eps +pypy/doc/statistic/*.pdf +pypy/translator/cli/src/pypylib.dll +pypy/translator/cli/src/query.exe +pypy/translator/cli/src/main.exe +lib_pypy/ctypes_config_cache/_*_cache.py +lib_pypy/ctypes_config_cache/_*_*_.py +pypy/translator/cli/query-descriptions +pypy/doc/discussion/*.html +pypy/doc/discussion/ +include/*.h +include/*.inl +pypy/doc/config/*.html +pypy/doc/config/style.css +pypy/doc/config/ +pypy/doc/jit/*.html +pypy/doc/jit/style.css +pypy/doc/image/lattice1.png +pypy/doc/image/lattice2.png +pypy/doc/image/lattice3.png +pypy/doc/image/stackless_informal.png +pypy/doc/image/parsing_example*.png From commits-noreply at bitbucket.org Mon Dec 13 17:58:22 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 10:58:22 -0600 (CST) Subject: [pypy-svn] pypy commit 1afa61e2bcb8: Add sys._mercurial attribute. Message-ID: <20101213165822.590581E12DA@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Dan Villiom Podlaski Christiansen # Date 1292254419 0 # Node ID 1afa61e2bcb88068ceabee8ccbe39a39d5019e64 # Parent 805815216ca7038c2ac1bc6cd620b28b96aef298 Add sys._mercurial attribute. Based on the equivalent patch to CPython 2.7: Example output: $ ./python.exe -c 'import sys; print sys.subversion; print sys._mercurial' ('CPython', 'branches/release27-maint', '') ('CPython', 'add-hg-build-id.diff qbase qtip tip', 'a40affe1a0c3') $ ./pypy-c -c 'import sys; print(sys.subversion); print(sys._mercurial)' ('PyPy', '', '0') ('PyPy', 'default', 'e5edda84e0bb') The Mercurial version detection calls out to 'hg' rather than parsing the Mercurial dirstate directly. There are two reasons for this: 1) The Mercurial dirstate is a binary file, and its format is an implementation detail of Mercurial. 2) Tags are used in the output, and the Mercurial logic for determining what tags to apply is fairly complex and often extended by extensions. --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -59,6 +59,7 @@ class Module(MixedModule): 'pypy_version_info' : 'version.get_pypy_version_info(space)', 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', + '_mercurial' : 'version.get_mercurial_info(space)', 'hexversion' : 'version.get_hexversion(space)', 'displayhook' : 'hook.displayhook', --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -84,6 +84,39 @@ def get_subversion_info(space): space.wrap(svnbranch), space.wrap(str(svn_revision()))]) +def get_mercurial_info(space): + '''Obtain Mercurial version information by invoking the 'hg' command.''' + # TODO: support extracting from .hg_archival.txt + import py + from subprocess import Popen, PIPE + + pypyroot = os.path.abspath(os.path.join(pypydir, '..')) + hgexe = py.path.local.sysfind('hg') + + if hgexe and os.path.isdir(os.path.join(pypyroot, '.hg')): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull + + p = Popen([str(hgexe), 'id', '-i', pypyroot], stdout=PIPE, env=env) + hgid = p.stdout.read().strip() + + p = Popen([str(hgexe), 'id', '-t', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip().split()[0] + + if hgtag == 'tip': + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip() + + return space.newtuple([space.wrap('PyPy'), + space.wrap(hgtag), + space.wrap(hgid)]) + else: + return space.w_None + def tuple2hex(ver): d = {'alpha': 0xA, 'beta': 0xB, From commits-noreply at bitbucket.org Mon Dec 13 17:58:23 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 10:58:23 -0600 (CST) Subject: [pypy-svn] pypy commit 5e8eb0237e7f: Whack at app_main until I get a result that I'm kind of happy with. Message-ID: <20101213165823.135F01E12DA@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Armin Rigo # Date 1292256686 0 # Node ID 5e8eb0237e7f098cfa76aa48cbc06aa44da7fceb # Parent 14922678ff991b78f469128b2e0ffc819897fb94 Whack at app_main until I get a result that I'm kind of happy with. Add tests for parse_command_line(). --- a/pypy/translator/goal/app_main.py +++ b/pypy/translator/goal/app_main.py @@ -22,6 +22,25 @@ DEBUG = False # dump exceptions be originalexcepthook = sys.__excepthook__ +def handle_sys_exit(e): + # exit if we catch a w_SystemExit + exitcode = e.code + if exitcode is None: + exitcode = 0 + else: + try: + exitcode = int(exitcode) + except: + # not an integer: print it to stderr + try: + stderr = sys.stderr + except AttributeError: + pass # too bad + else: + print >> stderr, exitcode + exitcode = 1 + raise SystemExit(exitcode) + def run_toplevel(f, *fargs, **fkwds): """Calls f() and handles all OperationErrors. Intended use is to run the main program or one interactive statement. @@ -45,72 +64,58 @@ def run_toplevel(f, *fargs, **fkwds): stdout.write('\n') except SystemExit, e: - # exit if we catch a w_SystemExit - exitcode = e.code - if exitcode is None: - exitcode = 0 - else: - try: - exitcode = int(exitcode) - except: - # not an integer: print it to stderr - try: - stderr = sys.stderr - except AttributeError: - pass # too bad - else: - print >> stderr, exitcode - exitcode = 1 - raise SystemExit(exitcode) + handle_sys_exit(e) + except: + display_exception() + return False + return True # success + +def display_exception(): + etype, evalue, etraceback = sys.exc_info() + try: + # extra debugging info in case the code below goes very wrong + if DEBUG and hasattr(sys, 'stderr'): + s = getattr(etype, '__name__', repr(etype)) + print >> sys.stderr, "debug: exception-type: ", s + print >> sys.stderr, "debug: exception-value:", str(evalue) + tbentry = etraceback + if tbentry: + while tbentry.tb_next: + tbentry = tbentry.tb_next + lineno = tbentry.tb_lineno + filename = tbentry.tb_frame.f_code.co_filename + print >> sys.stderr, "debug: exception-tb: %s:%d" % ( + filename, lineno) + + # set the sys.last_xxx attributes + sys.last_type = etype + sys.last_value = evalue + sys.last_traceback = etraceback + + # call sys.excepthook + hook = getattr(sys, 'excepthook', originalexcepthook) + hook(etype, evalue, etraceback) + return # done except: - etype, evalue, etraceback = sys.exc_info() try: - # extra debugging info in case the code below goes very wrong - if DEBUG and hasattr(sys, 'stderr'): - s = getattr(etype, '__name__', repr(etype)) - print >> sys.stderr, "debug: exception-type: ", s - print >> sys.stderr, "debug: exception-value:", str(evalue) - tbentry = etraceback - if tbentry: - while tbentry.tb_next: - tbentry = tbentry.tb_next - lineno = tbentry.tb_lineno - filename = tbentry.tb_frame.f_code.co_filename - print >> sys.stderr, "debug: exception-tb: %s:%d" % ( - filename, lineno) + stderr = sys.stderr + except AttributeError: + pass # too bad + else: + print >> stderr, 'Error calling sys.excepthook:' + originalexcepthook(*sys.exc_info()) + print >> stderr + print >> stderr, 'Original exception was:' - # set the sys.last_xxx attributes - sys.last_type = etype - sys.last_value = evalue - sys.last_traceback = etraceback + # we only get here if sys.excepthook didn't do its job + originalexcepthook(etype, evalue, etraceback) - # call sys.excepthook - hook = getattr(sys, 'excepthook', originalexcepthook) - hook(etype, evalue, etraceback) - return False # done - - except: - try: - stderr = sys.stderr - except AttributeError: - pass # too bad - else: - print >> stderr, 'Error calling sys.excepthook:' - originalexcepthook(*sys.exc_info()) - print >> stderr - print >> stderr, 'Original exception was:' - - # we only get here if sys.excepthook didn't do its job - originalexcepthook(etype, evalue, etraceback) - return False - - return True # success # ____________________________________________________________ # Option parsing -def print_info(): +def print_info(*args): try: options = sys.pypy_translation_info except AttributeError: @@ -120,15 +125,17 @@ def print_info(): optitems.sort() for name, value in optitems: print ' %51s: %s' % (name, value) + raise SystemExit -def print_help(): +def print_help(*args): print 'usage: %s [options]' % (sys.executable,) print __doc__.rstrip() if 'pypyjit' in sys.builtin_module_names: - print_jit_help() + _print_jit_help() print + raise SystemExit -def print_jit_help(): +def _print_jit_help(): import pypyjit items = pypyjit.defaults.items() items.sort() @@ -136,6 +143,18 @@ def print_jit_help(): print ' --jit %s=N %slow-level JIT parameter (default %s)' % ( key, ' '*(18-len(key)), value) +def print_version(*args): + print "Python", sys.version + raise SystemExit + +def set_jit_option(options, jitparam, *args): + if 'pypyjit' not in sys.builtin_module_names: + print >> sys.stderr, ("Warning: No jit support in %s" % + (sys.executable,)) + else: + import pypyjit + pypyjit.set_param(jitparam) + class CommandLineError(Exception): pass @@ -171,18 +190,6 @@ if 'nt' in sys.builtin_module_names: else: IS_WINDOWS = False -def get_argument(option, argv, i): - arg = argv[i] - n = len(option) - if len(arg) > n: - return arg[n:], i - else: - i += 1 - if i >= len(argv): - raise CommandLineError('Argument expected for the %s option' % - option) - return argv[i], i - def get_library_path(executable): search = executable while 1: @@ -200,7 +207,7 @@ def get_library_path(executable): break # found! return newpath -def setup_initial_paths(executable, nanos, readenv=True, **extra): +def setup_initial_paths(executable, nanos, ignore_environment=False, **extra): # a substituted os if we are translated global os os = nanos @@ -221,6 +228,7 @@ def setup_initial_paths(executable, nano sys.executable = os.path.abspath(executable) newpath = get_library_path(executable) + readenv = not ignore_environment path = readenv and os.getenv('PYTHONPATH') if path: newpath = path.split(os.pathsep) + newpath @@ -232,84 +240,181 @@ def setup_initial_paths(executable, nano sys.path.append(dir) _seen[dir] = True +# Order is significant! +sys_flags = ( + "debug", + "py3k_warning", + "division_warning", + "division_new", + "inspect", + "interactive", + "optimize", + "dont_write_bytecode", + "no_user_site", + "no_site", + "ignore_environment", + "tabcheck", + "verbose", + "unicode", + "bytes_warning", +) + + +default_options = dict.fromkeys( + sys_flags + + ("run_command", + "run_module", + "run_stdin", + "warnoptions", + "unbuffered"), 0) + + +PYTHON26 = False + +def simple_option(options, name, iterargv): + options[name] += 1 + +def div_option(options, div, iterargv): + if div == "warn": + options["division_warning"] = 1 + elif div == "warnall": + options["division_warning"] = 2 + elif div == "new": + options["division_new"] = 1 + elif div != "old": + raise CommandLineError("invalid division option: %r" % (div,)) + +def c_option(options, runcmd, iterargv): + options["run_command"] = runcmd + return ['-c'] + list(iterargv) + +def m_option(options, runmodule, iterargv): + options["run_module"] = True + return [runmodule] + list(iterargv) + +def W_option(options, warnoption, iterargv): + options["warnoptions"].append(warnoption) + +def end_options(options, _, iterargv): + return list(iterargv) + +cmdline_options = { + # simple options just increment the counter of the options listed above + 'd': (simple_option, 'debug'), + 'i': (simple_option, 'interactive'), + 'O': (simple_option, 'optimize'), + 'S': (simple_option, 'no_site'), + 'E': (simple_option, 'ignore_environment'), + 't': (simple_option, 'tabcheck'), + 'v': (simple_option, 'verbose'), + 'U': (simple_option, 'unicode'), + 'u': (simple_option, 'unbuffered'), + # more complex options + 'Q': (div_option, Ellipsis), + 'c': (c_option, Ellipsis), + 'm': (m_option, Ellipsis), + 'W': (W_option, Ellipsis), + 'V': (print_version, None), + '--version': (print_version, None), + '--info': (print_info, None), + 'h': (print_help, None), + '--help': (print_help, None), + '--jit': (set_jit_option, Ellipsis), + '--': (end_options, None), + } + +if PYTHON26: + cmdline_options.update({ + '3': (simple_option, 'py3k_warning'), + 'B': (simple_option, 'dont_write_bytecode'), + 's': (simple_option, 'no_user_site'), + 'b': (simple_option, 'bytes_warning'), + }) + + +def handle_argument(c, options, iterargv, iterarg=iter(())): + function, funcarg = cmdline_options[c] + # + # If needed, fill in the real argument by taking it from the command line + if funcarg is Ellipsis: + remaining = list(iterarg) + if remaining: + funcarg = ''.join(remaining) + else: + try: + funcarg = iterargv.next() + except StopIteration: + if len(c) == 1: + c = '-' + c + raise CommandLineError('Argument expected for the %r option' % c) + # + return function(options, funcarg, iterargv) + def parse_command_line(argv): - go_interactive = False - run_command = False - import_site = True - i = 0 - run_module = False - run_stdin = False - warnoptions = [] - unbuffered = False - readenv = True - while i < len(argv): - arg = argv[i] - if not arg.startswith('-'): - break - if arg == '-i': - go_interactive = True - elif arg.startswith('-c'): - cmd, i = get_argument('-c', argv, i) - argv[i] = '-c' - run_command = True - break - elif arg == '-E': - readenv = False - elif arg == '-u': - unbuffered = True - elif arg == '-O' or arg == '-OO': - pass - elif arg == '--version' or arg == '-V': - print "Python", sys.version - return - elif arg == '--info': - print_info() - return - elif arg == '-h' or arg == '--help': - print_help() - return - elif arg == '-S': - import_site = False - elif arg == '-': - run_stdin = True - break # not an option but a file name representing stdin - elif arg.startswith('-m'): - module, i = get_argument('-m', argv, i) - argv[i] = module - run_module = True - break - elif arg.startswith('-W'): - warnoptions, i = get_argument('-W', argv, i) - elif arg.startswith('--jit'): - jitparam, i = get_argument('--jit', argv, i) - if 'pypyjit' not in sys.builtin_module_names: - print >> sys.stderr, ("Warning: No jit support in %s" % - (sys.executable,)) - else: - import pypyjit - pypyjit.set_param(jitparam) - elif arg == '--': - i += 1 - break # terminates option list + options = default_options.copy() + options['warnoptions'] = [] + # + iterargv = iter(argv) + argv = None + for arg in iterargv: + # + # If the next argument isn't at least two characters long or + # doesn't start with '-', stop processing + if len(arg) < 2 or arg[0] != '-': + if IS_WINDOWS and arg == '/?': # special case + print_help() + argv = [arg] + list(iterargv) # finishes processing + # + # If the next argument is directly in cmdline_options, handle + # it as a single argument + elif arg in cmdline_options: + argv = handle_argument(arg, options, iterargv) + # + # Else interpret the rest of the argument character by character else: - raise CommandLineError('unrecognized option %r' % (arg,)) - i += 1 - sys.argv[:] = argv[i:] # don't change the list that sys.argv is bound to - if not sys.argv: # (relevant in case of "reload(sys)") - sys.argv.append('') - run_stdin = True - return locals() + iterarg = iter(arg) + iterarg.next() # skip the '-' + for c in iterarg: + if c not in cmdline_options: + raise CommandLineError('Unknown option: -%s' % (c,)) + argv = handle_argument(c, options, iterargv, iterarg) -def run_command_line(go_interactive, + if not argv: + argv = [''] + options["run_stdin"] = True + elif argv[0] == '-': + options["run_stdin"] = True + + # don't change the list that sys.argv is bound to + # (relevant in case of "reload(sys)") + sys.argv[:] = argv + + if (options["interactive"] or + (not options["ignore_environment"] and os.getenv('PYTHONINSPECT'))): + options["inspect"] = True + + if PYTHON26 and we_are_translated(): + flags = [options[flag] for flag in sys_flags] + sys.flags = type(sys.flags)(flags) + sys.py3kwarning = sys.flags.py3k_warning + +## if not we_are_translated(): +## for key in sorted(options): +## print '%40s: %s' % (key, options[key]) +## print '%40s: %s' % ("sys.argv", sys.argv) + + return options + +def run_command_line(interactive, + inspect, run_command, - import_site, + no_site, run_module, run_stdin, warnoptions, unbuffered, - readenv, - cmd=None, + ignore_environment, **ignored): # with PyPy in top of CPython we can only have around 100 # but we need more in the translated PyPy for the compiler package @@ -325,14 +430,18 @@ def run_command_line(go_interactive, mainmodule = type(sys)('__main__') sys.modules['__main__'] = mainmodule - if import_site: + if not no_site: try: import site except: print >> sys.stderr, "'import site' failed" + readenv = not ignore_environment + pythonwarnings = readenv and os.getenv('PYTHONWARNINGS') + if pythonwarnings: + warnoptions.extend(pythonwarnings.split(',')) if warnoptions: - sys.warnoptions.append(warnoptions) + sys.warnoptions[:] = warnoptions from warnings import _processoptions _processoptions(sys.warnoptions) @@ -352,26 +461,28 @@ def run_command_line(go_interactive, signal.signal(signal.SIGXFSZ, signal.SIG_IGN) def inspect_requested(): - # We get an interactive prompt in one of the following two cases: + # We get an interactive prompt in one of the following three cases: # - # * go_interactive=True, either from the "-i" option or - # from the fact that we printed the banner; + # * interactive=True, from the "-i" option + # or + # * inspect=True and stdin is a tty # or # * PYTHONINSPECT is set and stdin is a tty. # - return (go_interactive or - (readenv and os.getenv('PYTHONINSPECT') and sys.stdin.isatty())) + return (interactive or + ((inspect or (readenv and os.getenv('PYTHONINSPECT'))) + and sys.stdin.isatty())) success = True try: - if run_command: + if run_command != 0: # handle the "-c" command # Put '' on sys.path sys.path.insert(0, '') def run_it(): - exec cmd in mainmodule.__dict__ + exec run_command in mainmodule.__dict__ success = run_toplevel(run_it) elif run_module: # handle the "-m" command @@ -389,7 +500,7 @@ def run_command_line(go_interactive, # put it's directory on sys.path sys.path.insert(0, '') - if go_interactive or sys.stdin.isatty(): + if interactive or sys.stdin.isatty(): # If stdin is a tty or if "-i" is specified, we print # a banner and run $PYTHONSTARTUP. print_banner() @@ -410,7 +521,7 @@ def run_command_line(go_interactive, exec co_python_startup in mainmodule.__dict__ run_toplevel(run_it) # Then we need a prompt. - go_interactive = True + inspect = True else: # If not interactive, just read and execute stdin normally. def run_it(): @@ -426,14 +537,25 @@ def run_command_line(go_interactive, sys.path.insert(0, scriptdir) success = run_toplevel(execfile, sys.argv[0], mainmodule.__dict__) - # start a prompt if requested + except SystemExit, e: + status = e.code if inspect_requested(): + display_exception() + else: + status = not success + + # start a prompt if requested + if inspect_requested(): + inteactive = False + try: from _pypy_interact import interactive_console success = run_toplevel(interactive_console, mainmodule) - except SystemExit, e: - return e.code - else: - return not success + except SystemExit, e: + status = e.code + else: + status = not success + + return status def resolvedirof(filename): try: @@ -461,8 +583,6 @@ def entry_point(executable, argv, nanos) except CommandLineError, e: print_error(str(e)) return 2 - if cmdline is None: - return 0 setup_initial_paths(executable, nanos, **cmdline) return run_command_line(**cmdline) --- a/pypy/translator/goal/test2/test_app_main.py +++ b/pypy/translator/goal/test2/test_app_main.py @@ -43,6 +43,93 @@ crashing_demo_script = getscript(""" """) +class TestParseCommandLine: + + def check_options(self, options, sys_argv, **expected): + assert sys.argv == sys_argv + for key, value in expected.items(): + assert options[key] == value + for key, value in options.items(): + if key not in expected: + assert not value, ( + "option %r has unexpectedly the value %r" % (key, value)) + + def check(self, argv, **expected): + import StringIO + from pypy.translator.goal import app_main + saved_sys_argv = sys.argv[:] + saved_sys_stdout = sys.stdout + saved_sys_stderr = sys.stdout + app_main.os = os + try: + sys.stdout = sys.stderr = StringIO.StringIO() + try: + options = app_main.parse_command_line(argv) + except SystemExit: + output = expected['output_contains'] + assert output in sys.stdout.getvalue() + else: + self.check_options(options, **expected) + finally: + sys.argv[:] = saved_sys_argv + sys.stdout = saved_sys_stdout + sys.stderr = saved_sys_stderr + + def test_all_combinations_I_can_think_of(self): + self.check([], sys_argv=[''], run_stdin=True) + self.check(['-'], sys_argv=['-'], run_stdin=True) + self.check(['-S'], sys_argv=[''], run_stdin=True, no_site=1) + self.check(['-OO'], sys_argv=[''], run_stdin=True, optimize=2) + self.check(['-O', '-O'], sys_argv=[''], run_stdin=True, optimize=2) + self.check(['-Qnew'], sys_argv=[''], run_stdin=True, division_new=1) + self.check(['-Qold'], sys_argv=[''], run_stdin=True, division_new=0) + self.check(['-Qwarn'], sys_argv=[''], run_stdin=True, division_warning=1) + self.check(['-Qwarnall'], sys_argv=[''], run_stdin=True, + division_warning=2) + self.check(['-Q', 'new'], sys_argv=[''], run_stdin=True, division_new=1) + self.check(['-SOQnew'], sys_argv=[''], run_stdin=True, + no_site=1, optimize=1, division_new=1) + self.check(['-SOQ', 'new'], sys_argv=[''], run_stdin=True, + no_site=1, optimize=1, division_new=1) + self.check(['-i'], sys_argv=[''], run_stdin=True, + interactive=1, inspect=1) + self.check(['-h'], output_contains='usage:') + self.check(['-S', '-tO', '-h'], output_contains='usage:') + self.check(['-S', '-thO'], output_contains='usage:') + self.check(['-S', '-tO', '--help'], output_contains='usage:') + self.check(['-S', '-tO', '--info'], output_contains='translation') + self.check(['-S', '-tO', '--version'], output_contains='Python') + self.check(['-S', '-tOV'], output_contains='Python') + self.check(['--jit', 'foobar', '-S'], sys_argv=[''], + run_stdin=True, no_site=1) + self.check(['-c', 'pass'], sys_argv=['-c'], run_command='pass') + self.check(['-cpass'], sys_argv=['-c'], run_command='pass') + self.check(['-cpass','x'], sys_argv=['-c','x'], run_command='pass') + self.check(['-Sc', 'pass'], sys_argv=['-c'], run_command='pass', + no_site=1) + self.check(['-Scpass'], sys_argv=['-c'], run_command='pass', no_site=1) + self.check(['-c', '', ''], sys_argv=['-c', ''], run_command='') + self.check(['-mfoo', 'bar', 'baz'], sys_argv=['foo', 'bar', 'baz'], + run_module=True) + self.check(['-m', 'foo', 'bar', 'baz'], sys_argv=['foo', 'bar', 'baz'], + run_module=True) + self.check(['-Smfoo', 'bar', 'baz'], sys_argv=['foo', 'bar', 'baz'], + run_module=True, no_site=1) + self.check(['-Sm', 'foo', 'bar', 'baz'], sys_argv=['foo', 'bar', 'baz'], + run_module=True, no_site=1) + self.check(['-', 'foo', 'bar'], sys_argv=['-', 'foo', 'bar'], + run_stdin=True) + self.check(['foo', 'bar'], sys_argv=['foo', 'bar']) + self.check(['foo', '-i'], sys_argv=['foo', '-i']) + self.check(['-i', 'foo'], sys_argv=['foo'], interactive=1, inspect=1) + self.check(['--', 'foo'], sys_argv=['foo']) + self.check(['--', '-i', 'foo'], sys_argv=['-i', 'foo']) + self.check(['--', '-', 'foo'], sys_argv=['-', 'foo'], run_stdin=True) + self.check(['-Wbog'], sys_argv=[''], warnoptions=['bog'], run_stdin=True) + self.check(['-W', 'ab', '-SWc'], sys_argv=[''], warnoptions=['ab', 'c'], + run_stdin=True, no_site=1) + + class TestInteraction: """ These tests require pexpect (UNIX-only). @@ -416,7 +503,7 @@ class TestNonInteractive: def test_option_W_crashing(self): data = self.run('-W') - assert 'Argument expected for the -W option' in data + assert "Argument expected for the '-W' option" in data def test_option_W_arg_ignored(self): data = self.run('-Wc') From arigo at codespeak.net Mon Dec 13 18:02:02 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 18:02:02 +0100 (CET) Subject: [pypy-svn] r80039 - in pypy/trunk/pypy: jit/backend jit/backend/llsupport jit/backend/x86 jit/backend/x86/test jit/metainterp rlib rpython/lltypesystem translator/c/src Message-ID: <20101213170202.31F15282BEA@codespeak.net> Author: arigo Date: Mon Dec 13 18:01:51 2010 New Revision: 80039 Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py pypy/trunk/pypy/jit/backend/model.py pypy/trunk/pypy/jit/backend/x86/assembler.py pypy/trunk/pypy/jit/backend/x86/regalloc.py pypy/trunk/pypy/jit/backend/x86/rx86.py pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py pypy/trunk/pypy/jit/metainterp/pyjitpl.py pypy/trunk/pypy/rlib/rstack.py pypy/trunk/pypy/rpython/lltypesystem/lloperation.py pypy/trunk/pypy/translator/c/src/stack.h Log: Merge branch/jit-stackcheck. Reintroduces (cheap) stack checks in JITted code. Modified: pypy/trunk/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/trunk/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/trunk/pypy/jit/backend/llsupport/llmodel.py Mon Dec 13 18:01:51 2010 @@ -115,6 +115,7 @@ self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = lambda: (0, 0, 0) def _setup_exception_handling_translated(self): @@ -138,9 +139,20 @@ # in the assignment to self.saved_exc_value, as needed. self.saved_exc_value = exc_value + from pypy.rlib import rstack + STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def insert_stack_check(): + startaddr = rstack._stack_get_start_adr() + length = rstack._stack_get_length() + f = llhelper(STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + slowpathaddr = rffi.cast(lltype.Signed, f) + return startaddr, length, slowpathaddr + self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = insert_stack_check def _setup_on_leave_jitted_untranslated(self): # assume we don't need a backend leave in this case Modified: pypy/trunk/pypy/jit/backend/model.py ============================================================================== --- pypy/trunk/pypy/jit/backend/model.py (original) +++ pypy/trunk/pypy/jit/backend/model.py Mon Dec 13 18:01:51 2010 @@ -8,6 +8,7 @@ done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + exit_frame_with_exception_v = -1 total_compiled_loops = 0 total_compiled_bridges = 0 total_freed_loops = 0 Modified: pypy/trunk/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/assembler.py (original) +++ pypy/trunk/pypy/jit/backend/x86/assembler.py Mon Dec 13 18:01:51 2010 @@ -84,6 +84,7 @@ self.fail_boxes_count = 0 self._current_depths_cache = (0, 0) self.datablockwrapper = None + self.stack_check_slowpath_imm = imm0 self.teardown() def leave_jitted_hook(self): @@ -122,6 +123,7 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + self._build_stack_check_slowpath() debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') @@ -194,6 +196,82 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.malloc_fixedsize_slowpath2 = rawstart + _STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def _build_stack_check_slowpath(self): + from pypy.rlib import rstack + mc = codebuf.MachineCodeBlockWrapper() + mc.PUSH_r(ebp.value) + mc.MOV_rr(ebp.value, esp.value) + # + if IS_X86_64: + # on the x86_64, we have to save all the registers that may + # have been used to pass arguments + for reg in [edi, esi, edx, ecx, r8, r9]: + mc.PUSH_r(reg.value) + mc.SUB_ri(esp.value, 8*8) + for i in range(8): + mc.MOVSD_sx(8*i, i) # xmm0 to xmm7 + # + if IS_X86_32: + mc.LEA_rb(eax.value, +8) + mc.PUSH_r(eax.value) + elif IS_X86_64: + mc.LEA_rb(edi.value, +16) + mc.AND_ri(esp.value, -16) + # + f = llhelper(self._STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + addr = rffi.cast(lltype.Signed, f) + mc.CALL(imm(addr)) + # + mc.MOV(eax, heap(self.cpu.pos_exception())) + mc.TEST_rr(eax.value, eax.value) + mc.J_il8(rx86.Conditions['NZ'], 0) + jnz_location = mc.get_relative_pos() + # + if IS_X86_64: + # restore the registers + for i in range(7, -1, -1): + mc.MOVSD_xs(i, 8*i) + for i, reg in [(6, r9), (5, r8), (4, ecx), + (3, edx), (2, esi), (1, edi)]: + mc.MOV_rb(reg.value, -8*i) + # + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.RET() + # + # patch the JNZ above + offset = mc.get_relative_pos() - jnz_location + assert 0 < offset <= 127 + mc.overwrite(jnz_location-1, chr(offset)) + # clear the exception from the global position + mc.MOV(eax, heap(self.cpu.pos_exc_value())) + mc.MOV(heap(self.cpu.pos_exception()), imm0) + mc.MOV(heap(self.cpu.pos_exc_value()), imm0) + # save the current exception instance into fail_boxes_ptr[0] + adr = self.fail_boxes_ptr.get_addr_for_num(0) + mc.MOV(heap(adr), eax) + # call the helper function to set the GC flag on the fail_boxes_ptr + # array (note that there is no exception any more here) + addr = self.cpu.get_on_leave_jitted_int(save_exception=False) + mc.CALL(imm(addr)) + # + assert self.cpu.exit_frame_with_exception_v >= 0 + mc.MOV_ri(eax.value, self.cpu.exit_frame_with_exception_v) + # + # footer -- note the ADD, which skips the return address of this + # function, and will instead return to the caller's caller. Note + # also that we completely ignore the saved arguments, because we + # are interrupting the function. + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.ADD_ri(esp.value, WORD) + mc.RET() + # + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.stack_check_slowpath_imm = imm(rawstart) + def assemble_loop(self, inputargs, operations, looptoken, log): '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) @@ -468,6 +546,24 @@ for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) + def _call_header_with_stack_check(self): + startaddr, length, slowpathaddr = self.cpu.insert_stack_check() + if slowpathaddr == 0: + pass # no stack check (e.g. not translated) + else: + self.mc.MOV(eax, esp) # MOV eax, current + self.mc.SUB(eax, heap(startaddr)) # SUB eax, [startaddr] + self.mc.CMP(eax, imm(length)) # CMP eax, length + self.mc.J_il8(rx86.Conditions['B'], 0) # JB .skip + jb_location = self.mc.get_relative_pos() + self.mc.CALL(self.stack_check_slowpath_imm) # CALL slowpath + # patch the JB above # .skip: + offset = self.mc.get_relative_pos() - jb_location + assert 0 < offset <= 127 + self.mc.overwrite(jb_location-1, chr(offset)) + # + self._call_header() + def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -485,7 +581,7 @@ # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] @@ -526,7 +622,7 @@ unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones Modified: pypy/trunk/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/trunk/pypy/jit/backend/x86/regalloc.py Mon Dec 13 18:01:51 2010 @@ -101,7 +101,6 @@ return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): - exc = False def __init__(self, assembler, translate_support_code=False): assert isinstance(translate_support_code, bool) @@ -428,7 +427,10 @@ locs = [self.loc(op.getarg(i)) for i in range(op.numargs())] locs_are_ref = [op.getarg(i).type == REF for i in range(op.numargs())] fail_index = self.assembler.cpu.get_fail_descr_number(op.getdescr()) - self.assembler.generate_failure(fail_index, locs, self.exc, + # note: no exception should currently be set in llop.get_exception_addr + # even if this finish may be an exit_frame_with_exception (in this case + # the exception instance is in locs[0]). + self.assembler.generate_failure(fail_index, locs, False, locs_are_ref) self.possibly_free_vars_for_op(op) Modified: pypy/trunk/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/rx86.py (original) +++ pypy/trunk/pypy/jit/backend/x86/rx86.py Mon Dec 13 18:01:51 2010 @@ -347,6 +347,8 @@ INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') INSN_br = insn(rex_w, chr(base+1), register(2,8), stack_bp(1)) INSN_rb = insn(rex_w, chr(base+3), register(1,8), stack_bp(2)) + INSN_rm = insn(rex_w, chr(base+3), register(1,8), mem_reg_plus_const(2)) + INSN_rj = insn(rex_w, chr(base+3), register(1,8), '\x05', immediate(2)) INSN_bi8 = insn(rex_w, '\x83', orbyte(base), stack_bp(1), immediate(2,'b')) INSN_bi32= insn(rex_w, '\x81', orbyte(base), stack_bp(1), immediate(2)) @@ -364,7 +366,7 @@ INSN_bi32(mc, offset, immed) INSN_bi._always_inline_ = True # try to constant-fold single_byte() - return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br + return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br, INSN_rm, INSN_rj def select_8_or_32_bit_immed(insn_8, insn_32): def INSN(*args): @@ -442,24 +444,22 @@ # ------------------------------ Arithmetic ------------------------------ - ADD_ri, ADD_rr, ADD_rb, _, _ = common_modes(0) - OR_ri, OR_rr, OR_rb, _, _ = common_modes(1) - AND_ri, AND_rr, AND_rb, _, _ = common_modes(4) - SUB_ri, SUB_rr, SUB_rb, _, _ = common_modes(5) - XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) - CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) + ADD_ri, ADD_rr, ADD_rb, _, _, ADD_rm, ADD_rj = common_modes(0) + OR_ri, OR_rr, OR_rb, _, _, OR_rm, OR_rj = common_modes(1) + AND_ri, AND_rr, AND_rb, _, _, AND_rm, AND_rj = common_modes(4) + SUB_ri, SUB_rr, SUB_rb, _, _, SUB_rm, SUB_rj = common_modes(5) + XOR_ri, XOR_rr, XOR_rb, _, _, XOR_rm, XOR_rj = common_modes(6) + CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br, CMP_rm, CMP_rj = common_modes(7) CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32) - CMP_rm = insn(rex_w, '\x3B', register(1, 8), mem_reg_plus_const(2)) CMP_mr = insn(rex_w, '\x39', register(2, 8), mem_reg_plus_const(1)) CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) - CMP_rj = insn(rex_w, '\x3B', register(1, 8), '\x05', immediate(2)) CMP32_mi = insn(rex_nw, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) Modified: pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/trunk/pypy/jit/backend/x86/test/test_ztranslation.py Mon Dec 13 18:01:51 2010 @@ -93,6 +93,9 @@ assert res == expected def test_direct_assembler_call_translates(self): + """Test CALL_ASSEMBLER and the recursion limit""" + from pypy.rlib.rstackovf import StackOverflow + class Thing(object): def __init__(self, val): self.val = val @@ -135,9 +138,35 @@ i += 1 return frame.thing.val - res = self.meta_interp(main, [0], inline=True, + driver2 = JitDriver(greens = [], reds = ['n']) + + def main2(bound): + try: + while portal2(bound) == -bound+1: + bound *= 2 + except StackOverflow: + pass + return bound + + def portal2(n): + while True: + driver2.jit_merge_point(n=n) + n -= 1 + if n <= 0: + return n + n = portal2(n) + assert portal2(10) == -9 + + def mainall(codeno, bound): + return main(codeno) + main2(bound) + + res = self.meta_interp(mainall, [0, 1], inline=True, policy=StopAtXPolicy(change)) - assert res == main(0) + print hex(res) + assert res & 255 == main(0) + bound = res & ~255 + assert 1024 <= bound <= 131072 + assert bound & (bound-1) == 0 # a power of two class TestTranslationRemoveTypePtrX86(CCompiledMixin): Modified: pypy/trunk/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/trunk/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/trunk/pypy/jit/metainterp/pyjitpl.py Mon Dec 13 18:01:51 2010 @@ -1294,6 +1294,10 @@ num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) # + tokens = self.loop_tokens_exit_frame_with_exception_ref + num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) + self.cpu.exit_frame_with_exception_v = num + # self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): Modified: pypy/trunk/pypy/rlib/rstack.py ============================================================================== --- pypy/trunk/pypy/rlib/rstack.py (original) +++ pypy/trunk/pypy/rlib/rstack.py Mon Dec 13 18:01:51 2010 @@ -6,21 +6,22 @@ import inspect from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rarithmetic import r_uint +from pypy.rlib import rgc from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.controllerentry import Controller, SomeControlledInstance from pypy.translator.tool.cbuild import ExternalCompilationInfo def stack_unwind(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_unwind(lltype.Void) raise RuntimeError("cannot unwind stack in non-translated versions") def stack_capture(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop ptr = llop.stack_capture(OPAQUE_STATE_HEADER_PTR) return frame_stack_top_controller.box(ptr) raise RuntimeError("cannot unwind stack in non-translated versions") @@ -28,26 +29,57 @@ def stack_frames_depth(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_frames_depth(lltype.Signed) else: return len(inspect.stack()) +# ____________________________________________________________ + compilation_info = ExternalCompilationInfo(includes=['src/stack.h']) -stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, - compilation_info=compilation_info, - _nowrapper=True, - _callable=lambda: _zero, - sandboxsafe=True) -_zero = rffi.cast(rffi.INT, 0) +def llexternal(name, args, res): + return rffi.llexternal(name, args, res, compilation_info=compilation_info, + sandboxsafe=True, _nowrapper=True) + +_stack_get_start = llexternal('LL_stack_get_start', [], lltype.Signed) +_stack_get_length = llexternal('LL_stack_get_length', [], lltype.Signed) +_stack_too_big_slowpath = llexternal('LL_stack_too_big_slowpath', + [lltype.Signed], lltype.Char) +# the following is used by the JIT +_stack_get_start_adr = llexternal('LL_stack_get_start_adr', [], lltype.Signed) + def stack_check(): - if rffi.cast(lltype.Signed, stack_too_big()): + if not we_are_translated(): + return + # + # Load the "current" stack position, or at least some address that + # points close to the current stack head + current = llop.stack_current(lltype.Signed) + # + # Load these variables from C code + start = _stack_get_start() + length = _stack_get_length() + # + # Common case: if 'current' is within [start:start+length], everything + # is fine + ofs = r_uint(current - start) + if ofs < r_uint(length): + return + # + # Else call the slow path + stack_check_slowpath(current) +stack_check._always_inline_ = True + + at rgc.no_collect +def stack_check_slowpath(current): + if ord(_stack_too_big_slowpath(current)): + # Now we are sure that the stack is really too big. Note that the # stack_unwind implementation is different depending on if stackless # is enabled. If it is it unwinds the stack, otherwise it simply # raises a RuntimeError. stack_unwind() +stack_check_slowpath._dont_inline_ = True # ____________________________________________________________ Modified: pypy/trunk/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/trunk/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/trunk/pypy/rpython/lltypesystem/lloperation.py Mon Dec 13 18:01:51 2010 @@ -526,6 +526,8 @@ 'get_stack_depth_limit':LLOp(sideeffects=False), 'set_stack_depth_limit':LLOp(), + 'stack_current': LLOp(sideeffects=False), + # __________ misc operations __________ 'keepalive': LLOp(), Modified: pypy/trunk/pypy/translator/c/src/stack.h ============================================================================== --- pypy/trunk/pypy/translator/c/src/stack.h (original) +++ pypy/trunk/pypy/translator/c/src/stack.h Mon Dec 13 18:01:51 2010 @@ -11,27 +11,17 @@ * It is needed to have RPyThreadStaticTLS, too. */ #include "thread.h" +extern char *_LLstacktoobig_stack_start; + void LL_stack_unwind(void); -int LL_stack_too_big_slowpath(void); +char LL_stack_too_big_slowpath(long); /* returns 0 (ok) or 1 (too big) */ -extern volatile char *_LLstacktoobig_stack_base_pointer; -extern long _LLstacktoobig_stack_min; -extern long _LLstacktoobig_stack_max; +/* some macros referenced from pypy.rlib.rstack */ +#define OP_STACK_CURRENT(r) r = (long)&r +#define LL_stack_get_start() ((long)_LLstacktoobig_stack_start) +#define LL_stack_get_length() MAX_STACK_SIZE +#define LL_stack_get_start_adr() ((long)&_LLstacktoobig_stack_start) /* JIT */ -static int LL_stack_too_big(void) -{ - /* The fast path of stack_too_big, called extremely often. - Making it static makes an *inlinable* copy of this small - function's implementation in each compilation unit. */ - char local; - long diff = &local - _LLstacktoobig_stack_base_pointer; - /* common case: we are still in the same thread as last time - we checked, and still in the allowed part of the stack */ - return ((diff < _LLstacktoobig_stack_min || - diff > _LLstacktoobig_stack_max) - /* if not, call the slow path */ - && LL_stack_too_big_slowpath()); -} #ifdef __GNUC__ # define PYPY_INHIBIT_TAIL_CALL() asm("/* inhibit_tail_call */") @@ -61,68 +51,75 @@ return &local - parent; } -volatile char *_LLstacktoobig_stack_base_pointer = NULL; -long _LLstacktoobig_stack_min = 0; -long _LLstacktoobig_stack_max = 0; -RPyThreadStaticTLS _LLstacktoobig_stack_base_pointer_key; +char *_LLstacktoobig_stack_start = NULL; +int stack_direction = 0; +RPyThreadStaticTLS start_tls_key; -int LL_stack_too_big_slowpath(void) +char LL_stack_too_big_slowpath(long current) { - char local; long diff; - char *baseptr; - /* Check that the stack is less than MAX_STACK_SIZE bytes bigger - than the value recorded in stack_base_pointer. The base - pointer is updated to the current value if it is still NULL - or if we later find a &local that is below it. The real - stack base pointer is stored in thread-local storage, but we - try to minimize its overhead by keeping a local copy in - stack_pointer_pointer. */ + char *baseptr, *curptr = (char*)current; + + /* The stack_start variable is updated to match the current value + if it is still 0 or if we later find a 'curptr' position + that is below it. The real stack_start pointer is stored in + thread-local storage, but we try to minimize its overhead by + keeping a local copy in _LLstacktoobig_stack_start. */ - if (_LLstacktoobig_stack_min == _LLstacktoobig_stack_max /* == 0 */) { + if (stack_direction == 0) { /* not initialized */ /* XXX We assume that initialization is performed early, when there is still only one thread running. This allows us to ignore race conditions here */ - char *errmsg = RPyThreadStaticTLS_Create( - &_LLstacktoobig_stack_base_pointer_key); + char *errmsg = RPyThreadStaticTLS_Create(&start_tls_key); if (errmsg) { /* XXX should we exit the process? */ fprintf(stderr, "Internal PyPy error: %s\n", errmsg); return 1; } if (_LL_stack_growing_direction(NULL) > 0) - _LLstacktoobig_stack_max = MAX_STACK_SIZE; + stack_direction = +1; else - _LLstacktoobig_stack_min = -MAX_STACK_SIZE; + stack_direction = -1; } - baseptr = (char *) RPyThreadStaticTLS_Get( - _LLstacktoobig_stack_base_pointer_key); + baseptr = (char *) RPyThreadStaticTLS_Get(start_tls_key); if (baseptr != NULL) { - diff = &local - baseptr; - if (_LLstacktoobig_stack_min <= diff && - diff <= _LLstacktoobig_stack_max) { - /* within bounds */ - _LLstacktoobig_stack_base_pointer = baseptr; + diff = curptr - baseptr; + if (((unsigned long)diff) < (unsigned long)MAX_STACK_SIZE) { + /* within bounds, probably just had a thread switch */ + _LLstacktoobig_stack_start = baseptr; return 0; } - if ((_LLstacktoobig_stack_min == 0 && diff < 0) || - (_LLstacktoobig_stack_max == 0 && diff > 0)) { - /* we underflowed the stack, which means that - the initial estimation of the stack base must - be revised (see below) */ + if (stack_direction > 0) { + if (diff < 0 && diff > -MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } else { - return 1; /* stack overflow */ + if (diff >= MAX_STACK_SIZE && diff < 2*MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } + /* else we underflowed the stack, which means that + the initial estimation of the stack base must + be revised */ } /* update the stack base pointer to the current value */ - baseptr = &local; - RPyThreadStaticTLS_Set(_LLstacktoobig_stack_base_pointer_key, baseptr); - _LLstacktoobig_stack_base_pointer = baseptr; + if (stack_direction > 0) { + /* the valid range is [curptr:curptr+MAX_STACK_SIZE] */ + baseptr = curptr; + } + else { + /* the valid range is [curptr-MAX_STACK_SIZE+1:curptr+1] */ + baseptr = curptr - MAX_STACK_SIZE + 1; + } + RPyThreadStaticTLS_Set(start_tls_key, baseptr); + _LLstacktoobig_stack_start = baseptr; return 0; } From arigo at codespeak.net Mon Dec 13 18:02:10 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 18:02:10 +0100 (CET) Subject: [pypy-svn] r80040 - pypy/branch/jit-stackcheck Message-ID: <20101213170210.7C9F3282BF0@codespeak.net> Author: arigo Date: Mon Dec 13 18:02:06 2010 New Revision: 80040 Removed: pypy/branch/jit-stackcheck/ Log: Branch merged. From commits-noreply at bitbucket.org Mon Dec 13 18:05:28 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 11:05:28 -0600 (CST) Subject: [pypy-svn] buildbot commit a7823765f22f: pull from the official bitbucket repo Message-ID: <20101213170528.197C7241416@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User Antonio Cuni # Date 1292259921 -3600 # Node ID a7823765f22f7c3385b6392ab3744055d3eac61e # Parent 1ea28ce0556aa3996303a6e6fdbf206218263f45 pull from the official bitbucket repo --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -117,7 +117,7 @@ def setup_steps(platform, factory, workd ## defaultBranch="trunk", ## workdir=workdir)) import getpass - repourl = 'https://bitbucket.org/pypy/pypy-tentative/' + repourl = 'https://bitbucket.org/pypy/pypy/' if getpass.getuser() == 'antocuni': # for debugging repourl = '/home/antocuni/pypy/pypy-hg' From fijal at codespeak.net Mon Dec 13 18:15:27 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Mon, 13 Dec 2010 18:15:27 +0100 (CET) Subject: [pypy-svn] r80041 - in pypy/branch/out-of-line-guards/pypy: jit/backend/llgraph jit/codewriter jit/metainterp rpython/lltypesystem Message-ID: <20101213171527.37FB1282BEA@codespeak.net> Author: fijal Date: Mon Dec 13 18:15:25 2010 New Revision: 80041 Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py Log: Progress on out-of-line guards. Use weakref for storing affected codes and cheat (horribly) to pretend that LoopToken is a low-level pointer Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py Mon Dec 13 18:15:25 2010 @@ -490,6 +490,13 @@ def get_invalidate_asm(self, TP, fieldname): def invalidate_asm(arg, fieldname): + prev = getattr(arg, fieldname) + next = prev + while next: + prev = next + x = llmemory.weakref_deref(history.LoopToken._TYPE, + prev.address) + next = next.next XXX # write me return invalidate_asm Modified: pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/codewriter/jtransform.py Mon Dec 13 18:15:25 2010 @@ -605,7 +605,8 @@ next = next.next prev.next = new_asmcode - args_s = [lltype_to_annotation(llmemory.GCREF)] * 2 + args_s = [lltype_to_annotation(llmemory.GCREF), + lltype_to_annotation(llmemory.WeakRefPtr)] s_result = lltype_to_annotation(lltype.Void) mixlevelann = MixLevelHelperAnnotator(self.cpu.rtyper) c_appender = mixlevelann.constfunc(appender, args_s, s_result) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py Mon Dec 13 18:15:25 2010 @@ -725,6 +725,20 @@ was compiled; but the LoopDescr remains alive and points to the generated assembler. """ + + _TYPE = lltype.Ptr(lltype.GcStruct('dummy struct for tests')) + + def _normalizedcontainer(self): + return self + + def _getobj(self, check=True): + return self + _as_ptr = _getobj + _obj0 = None + _obj = property(_getobj) + def _was_freed(self): + return False + terminating = False # see TerminatingLoopToken in compile.py outermost_jitdriver_sd = None # specnodes = ... Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py Mon Dec 13 18:15:25 2010 @@ -1,6 +1,5 @@ import py, os, sys from pypy.rpython.lltypesystem import lltype, llmemory, rclass -from pypy.rpython.lltypesystem.rclass import ASMCODE from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -2238,13 +2237,14 @@ op = op.copy_and_change(rop.CALL_ASSEMBLER, args=args, descr=token) self.history.operations.append(op) - def remember_jit_invariants(self, token): - lltoken = lltype.cast_opaque_ptr(llmemory.GCREF, token) + def remember_jit_invariants(self, loop): + lltoken_weakref = llmemory.weakref_create(loop.token) seen = {} for b_struct, c_appender in self.invariant_structs: if (b_struct, c_appender) not in seen: - heaptracker.int2adr(c_func.value).ptr(b_struct.value, lltoken) - seend[(b_struct, c_appender)] = None + heaptracker.int2adr(c_appender.value).ptr(b_struct.value, + lltoken_weakref) + seen[(b_struct, c_appender)] = None # ____________________________________________________________ Modified: pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py (original) +++ pypy/branch/out-of-line-guards/pypy/rpython/lltypesystem/rclass.py Mon Dec 13 18:15:25 2010 @@ -87,7 +87,7 @@ # a linked-list of assembler codes to invalidate in case jit_invariant_fields # are modified ASMCODE = lltype.GcForwardReference() -ASMCODE.become(GcStruct('asmcode', ('address', llmemory.GCREF), +ASMCODE.become(GcStruct('asmcode', ('address', llmemory.WeakRefPtr), ('next', lltype.Ptr(ASMCODE)))) def cast_vtable_to_typeptr(vtable): From arigo at codespeak.net Mon Dec 13 18:21:20 2010 From: arigo at codespeak.net (arigo at codespeak.net) Date: Mon, 13 Dec 2010 18:21:20 +0100 (CET) Subject: [pypy-svn] r80042 - pypy/branch/getopt-appmain/pypy/translator/goal Message-ID: <20101213172120.8606450818@codespeak.net> Author: arigo Date: Mon Dec 13 18:21:17 2010 New Revision: 80042 Modified: pypy/branch/getopt-appmain/pypy/translator/goal/app_main.py Log: Fix. Modified: pypy/branch/getopt-appmain/pypy/translator/goal/app_main.py ============================================================================== --- pypy/branch/getopt-appmain/pypy/translator/goal/app_main.py (original) +++ pypy/branch/getopt-appmain/pypy/translator/goal/app_main.py Mon Dec 13 18:21:17 2010 @@ -207,10 +207,7 @@ break # found! return newpath -def setup_initial_paths(executable, nanos, ignore_environment=False, **extra): - # a substituted os if we are translated - global os - os = nanos +def setup_initial_paths(executable, ignore_environment=False, **extra): # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'): @@ -578,12 +575,15 @@ '"license" for more information.') def entry_point(executable, argv, nanos): + # a substituted os if we are translated + global os + os = nanos try: cmdline = parse_command_line(argv) except CommandLineError, e: print_error(str(e)) return 2 - setup_initial_paths(executable, nanos, **cmdline) + setup_initial_paths(executable, **cmdline) return run_command_line(**cmdline) From commits-noreply at bitbucket.org Mon Dec 13 18:21:31 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 11:21:31 -0600 (CST) Subject: [pypy-svn] pypy commit 7bacafd654eb: closed branch jit-stackcheck Message-ID: <20101213172131.E1A4B2414ED@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User convert-repo # Date 1292245978 0 # Node ID 7bacafd654eb6220a097615f5389d1f6b68d9a75 # Parent db8930c0c92c602b50c7728fd2841f67eb2376e2 closed branch jit-stackcheck From commits-noreply at bitbucket.org Mon Dec 13 18:21:31 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 11:21:31 -0600 (CST) Subject: [pypy-svn] pypy commit 74bbe61137d9: Merge branch/jit-stackcheck. Message-ID: <20101213172131.438E52414EC@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Armin Rigo # Date 1292259711 0 # Node ID 74bbe61137d921345adafcb80d004ce7d8860ace # Parent 1afa61e2bcb88068ceabee8ccbe39a39d5019e64 Merge branch/jit-stackcheck. Reintroduces (cheap) stack checks in JITted code. --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -526,6 +526,8 @@ LL_OPERATIONS = { 'get_stack_depth_limit':LLOp(sideeffects=False), 'set_stack_depth_limit':LLOp(), + 'stack_current': LLOp(sideeffects=False), + # __________ misc operations __________ 'keepalive': LLOp(), --- a/pypy/jit/backend/x86/test/test_ztranslation.py +++ b/pypy/jit/backend/x86/test/test_ztranslation.py @@ -93,6 +93,9 @@ class TestTranslationX86(CCompiledMixin) assert res == expected def test_direct_assembler_call_translates(self): + """Test CALL_ASSEMBLER and the recursion limit""" + from pypy.rlib.rstackovf import StackOverflow + class Thing(object): def __init__(self, val): self.val = val @@ -135,9 +138,35 @@ class TestTranslationX86(CCompiledMixin) i += 1 return frame.thing.val - res = self.meta_interp(main, [0], inline=True, + driver2 = JitDriver(greens = [], reds = ['n']) + + def main2(bound): + try: + while portal2(bound) == -bound+1: + bound *= 2 + except StackOverflow: + pass + return bound + + def portal2(n): + while True: + driver2.jit_merge_point(n=n) + n -= 1 + if n <= 0: + return n + n = portal2(n) + assert portal2(10) == -9 + + def mainall(codeno, bound): + return main(codeno) + main2(bound) + + res = self.meta_interp(mainall, [0, 1], inline=True, policy=StopAtXPolicy(change)) - assert res == main(0) + print hex(res) + assert res & 255 == main(0) + bound = res & ~255 + assert 1024 <= bound <= 131072 + assert bound & (bound-1) == 0 # a power of two class TestTranslationRemoveTypePtrX86(CCompiledMixin): --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1294,6 +1294,10 @@ class MetaInterpStaticData(object): num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) # + tokens = self.loop_tokens_exit_frame_with_exception_ref + num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) + self.cpu.exit_frame_with_exception_v = num + # self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -101,7 +101,6 @@ class X86FrameManager(FrameManager): return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): - exc = False def __init__(self, assembler, translate_support_code=False): assert isinstance(translate_support_code, bool) @@ -428,7 +427,10 @@ class RegAlloc(object): locs = [self.loc(op.getarg(i)) for i in range(op.numargs())] locs_are_ref = [op.getarg(i).type == REF for i in range(op.numargs())] fail_index = self.assembler.cpu.get_fail_descr_number(op.getdescr()) - self.assembler.generate_failure(fail_index, locs, self.exc, + # note: no exception should currently be set in llop.get_exception_addr + # even if this finish may be an exit_frame_with_exception (in this case + # the exception instance is in locs[0]). + self.assembler.generate_failure(fail_index, locs, False, locs_are_ref) self.possibly_free_vars_for_op(op) --- a/pypy/jit/backend/model.py +++ b/pypy/jit/backend/model.py @@ -8,6 +8,7 @@ class AbstractCPU(object): done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + exit_frame_with_exception_v = -1 total_compiled_loops = 0 total_compiled_bridges = 0 total_freed_loops = 0 --- a/pypy/translator/c/src/stack.h +++ b/pypy/translator/c/src/stack.h @@ -11,27 +11,17 @@ * It is needed to have RPyThreadStaticTLS, too. */ #include "thread.h" +extern char *_LLstacktoobig_stack_start; + void LL_stack_unwind(void); -int LL_stack_too_big_slowpath(void); +char LL_stack_too_big_slowpath(long); /* returns 0 (ok) or 1 (too big) */ -extern volatile char *_LLstacktoobig_stack_base_pointer; -extern long _LLstacktoobig_stack_min; -extern long _LLstacktoobig_stack_max; +/* some macros referenced from pypy.rlib.rstack */ +#define OP_STACK_CURRENT(r) r = (long)&r +#define LL_stack_get_start() ((long)_LLstacktoobig_stack_start) +#define LL_stack_get_length() MAX_STACK_SIZE +#define LL_stack_get_start_adr() ((long)&_LLstacktoobig_stack_start) /* JIT */ -static int LL_stack_too_big(void) -{ - /* The fast path of stack_too_big, called extremely often. - Making it static makes an *inlinable* copy of this small - function's implementation in each compilation unit. */ - char local; - long diff = &local - _LLstacktoobig_stack_base_pointer; - /* common case: we are still in the same thread as last time - we checked, and still in the allowed part of the stack */ - return ((diff < _LLstacktoobig_stack_min || - diff > _LLstacktoobig_stack_max) - /* if not, call the slow path */ - && LL_stack_too_big_slowpath()); -} #ifdef __GNUC__ # define PYPY_INHIBIT_TAIL_CALL() asm("/* inhibit_tail_call */") @@ -61,68 +51,75 @@ long PYPY_NOINLINE _LL_stack_growing_dir return &local - parent; } -volatile char *_LLstacktoobig_stack_base_pointer = NULL; -long _LLstacktoobig_stack_min = 0; -long _LLstacktoobig_stack_max = 0; -RPyThreadStaticTLS _LLstacktoobig_stack_base_pointer_key; +char *_LLstacktoobig_stack_start = NULL; +int stack_direction = 0; +RPyThreadStaticTLS start_tls_key; -int LL_stack_too_big_slowpath(void) +char LL_stack_too_big_slowpath(long current) { - char local; long diff; - char *baseptr; - /* Check that the stack is less than MAX_STACK_SIZE bytes bigger - than the value recorded in stack_base_pointer. The base - pointer is updated to the current value if it is still NULL - or if we later find a &local that is below it. The real - stack base pointer is stored in thread-local storage, but we - try to minimize its overhead by keeping a local copy in - stack_pointer_pointer. */ + char *baseptr, *curptr = (char*)current; - if (_LLstacktoobig_stack_min == _LLstacktoobig_stack_max /* == 0 */) { + /* The stack_start variable is updated to match the current value + if it is still 0 or if we later find a 'curptr' position + that is below it. The real stack_start pointer is stored in + thread-local storage, but we try to minimize its overhead by + keeping a local copy in _LLstacktoobig_stack_start. */ + + if (stack_direction == 0) { /* not initialized */ /* XXX We assume that initialization is performed early, when there is still only one thread running. This allows us to ignore race conditions here */ - char *errmsg = RPyThreadStaticTLS_Create( - &_LLstacktoobig_stack_base_pointer_key); + char *errmsg = RPyThreadStaticTLS_Create(&start_tls_key); if (errmsg) { /* XXX should we exit the process? */ fprintf(stderr, "Internal PyPy error: %s\n", errmsg); return 1; } if (_LL_stack_growing_direction(NULL) > 0) - _LLstacktoobig_stack_max = MAX_STACK_SIZE; + stack_direction = +1; else - _LLstacktoobig_stack_min = -MAX_STACK_SIZE; + stack_direction = -1; } - baseptr = (char *) RPyThreadStaticTLS_Get( - _LLstacktoobig_stack_base_pointer_key); + baseptr = (char *) RPyThreadStaticTLS_Get(start_tls_key); if (baseptr != NULL) { - diff = &local - baseptr; - if (_LLstacktoobig_stack_min <= diff && - diff <= _LLstacktoobig_stack_max) { - /* within bounds */ - _LLstacktoobig_stack_base_pointer = baseptr; + diff = curptr - baseptr; + if (((unsigned long)diff) < (unsigned long)MAX_STACK_SIZE) { + /* within bounds, probably just had a thread switch */ + _LLstacktoobig_stack_start = baseptr; return 0; } - if ((_LLstacktoobig_stack_min == 0 && diff < 0) || - (_LLstacktoobig_stack_max == 0 && diff > 0)) { - /* we underflowed the stack, which means that - the initial estimation of the stack base must - be revised (see below) */ + if (stack_direction > 0) { + if (diff < 0 && diff > -MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } else { - return 1; /* stack overflow */ + if (diff >= MAX_STACK_SIZE && diff < 2*MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } + /* else we underflowed the stack, which means that + the initial estimation of the stack base must + be revised */ } /* update the stack base pointer to the current value */ - baseptr = &local; - RPyThreadStaticTLS_Set(_LLstacktoobig_stack_base_pointer_key, baseptr); - _LLstacktoobig_stack_base_pointer = baseptr; + if (stack_direction > 0) { + /* the valid range is [curptr:curptr+MAX_STACK_SIZE] */ + baseptr = curptr; + } + else { + /* the valid range is [curptr-MAX_STACK_SIZE+1:curptr+1] */ + baseptr = curptr - MAX_STACK_SIZE + 1; + } + RPyThreadStaticTLS_Set(start_tls_key, baseptr); + _LLstacktoobig_stack_start = baseptr; return 0; } --- a/pypy/jit/backend/x86/rx86.py +++ b/pypy/jit/backend/x86/rx86.py @@ -347,6 +347,8 @@ def common_modes(group): INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') INSN_br = insn(rex_w, chr(base+1), register(2,8), stack_bp(1)) INSN_rb = insn(rex_w, chr(base+3), register(1,8), stack_bp(2)) + INSN_rm = insn(rex_w, chr(base+3), register(1,8), mem_reg_plus_const(2)) + INSN_rj = insn(rex_w, chr(base+3), register(1,8), '\x05', immediate(2)) INSN_bi8 = insn(rex_w, '\x83', orbyte(base), stack_bp(1), immediate(2,'b')) INSN_bi32= insn(rex_w, '\x81', orbyte(base), stack_bp(1), immediate(2)) @@ -364,7 +366,7 @@ def common_modes(group): INSN_bi32(mc, offset, immed) INSN_bi._always_inline_ = True # try to constant-fold single_byte() - return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br + return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br, INSN_rm, INSN_rj def select_8_or_32_bit_immed(insn_8, insn_32): def INSN(*args): @@ -442,24 +444,22 @@ class AbstractX86CodeBuilder(object): # ------------------------------ Arithmetic ------------------------------ - ADD_ri, ADD_rr, ADD_rb, _, _ = common_modes(0) - OR_ri, OR_rr, OR_rb, _, _ = common_modes(1) - AND_ri, AND_rr, AND_rb, _, _ = common_modes(4) - SUB_ri, SUB_rr, SUB_rb, _, _ = common_modes(5) - XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) - CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) + ADD_ri, ADD_rr, ADD_rb, _, _, ADD_rm, ADD_rj = common_modes(0) + OR_ri, OR_rr, OR_rb, _, _, OR_rm, OR_rj = common_modes(1) + AND_ri, AND_rr, AND_rb, _, _, AND_rm, AND_rj = common_modes(4) + SUB_ri, SUB_rr, SUB_rb, _, _, SUB_rm, SUB_rj = common_modes(5) + XOR_ri, XOR_rr, XOR_rb, _, _, XOR_rm, XOR_rj = common_modes(6) + CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br, CMP_rm, CMP_rj = common_modes(7) CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32) - CMP_rm = insn(rex_w, '\x3B', register(1, 8), mem_reg_plus_const(2)) CMP_mr = insn(rex_w, '\x39', register(2, 8), mem_reg_plus_const(1)) CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) - CMP_rj = insn(rex_w, '\x3B', register(1, 8), '\x05', immediate(2)) CMP32_mi = insn(rex_nw, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) --- a/pypy/rlib/rstack.py +++ b/pypy/rlib/rstack.py @@ -6,21 +6,22 @@ RPython-compliant way, intended mostly f import inspect from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rarithmetic import r_uint +from pypy.rlib import rgc from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.controllerentry import Controller, SomeControlledInstance from pypy.translator.tool.cbuild import ExternalCompilationInfo def stack_unwind(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_unwind(lltype.Void) raise RuntimeError("cannot unwind stack in non-translated versions") def stack_capture(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop ptr = llop.stack_capture(OPAQUE_STATE_HEADER_PTR) return frame_stack_top_controller.box(ptr) raise RuntimeError("cannot unwind stack in non-translated versions") @@ -28,26 +29,57 @@ def stack_capture(): def stack_frames_depth(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_frames_depth(lltype.Signed) else: return len(inspect.stack()) +# ____________________________________________________________ + compilation_info = ExternalCompilationInfo(includes=['src/stack.h']) -stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, - compilation_info=compilation_info, - _nowrapper=True, - _callable=lambda: _zero, - sandboxsafe=True) -_zero = rffi.cast(rffi.INT, 0) +def llexternal(name, args, res): + return rffi.llexternal(name, args, res, compilation_info=compilation_info, + sandboxsafe=True, _nowrapper=True) + +_stack_get_start = llexternal('LL_stack_get_start', [], lltype.Signed) +_stack_get_length = llexternal('LL_stack_get_length', [], lltype.Signed) +_stack_too_big_slowpath = llexternal('LL_stack_too_big_slowpath', + [lltype.Signed], lltype.Char) +# the following is used by the JIT +_stack_get_start_adr = llexternal('LL_stack_get_start_adr', [], lltype.Signed) + def stack_check(): - if rffi.cast(lltype.Signed, stack_too_big()): + if not we_are_translated(): + return + # + # Load the "current" stack position, or at least some address that + # points close to the current stack head + current = llop.stack_current(lltype.Signed) + # + # Load these variables from C code + start = _stack_get_start() + length = _stack_get_length() + # + # Common case: if 'current' is within [start:start+length], everything + # is fine + ofs = r_uint(current - start) + if ofs < r_uint(length): + return + # + # Else call the slow path + stack_check_slowpath(current) +stack_check._always_inline_ = True + + at rgc.no_collect +def stack_check_slowpath(current): + if ord(_stack_too_big_slowpath(current)): + # Now we are sure that the stack is really too big. Note that the # stack_unwind implementation is different depending on if stackless # is enabled. If it is it unwinds the stack, otherwise it simply # raises a RuntimeError. stack_unwind() +stack_check_slowpath._dont_inline_ = True # ____________________________________________________________ --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -84,6 +84,7 @@ class Assembler386(object): self.fail_boxes_count = 0 self._current_depths_cache = (0, 0) self.datablockwrapper = None + self.stack_check_slowpath_imm = imm0 self.teardown() def leave_jitted_hook(self): @@ -122,6 +123,7 @@ class Assembler386(object): self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + self._build_stack_check_slowpath() debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') @@ -194,6 +196,82 @@ class Assembler386(object): rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.malloc_fixedsize_slowpath2 = rawstart + _STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def _build_stack_check_slowpath(self): + from pypy.rlib import rstack + mc = codebuf.MachineCodeBlockWrapper() + mc.PUSH_r(ebp.value) + mc.MOV_rr(ebp.value, esp.value) + # + if IS_X86_64: + # on the x86_64, we have to save all the registers that may + # have been used to pass arguments + for reg in [edi, esi, edx, ecx, r8, r9]: + mc.PUSH_r(reg.value) + mc.SUB_ri(esp.value, 8*8) + for i in range(8): + mc.MOVSD_sx(8*i, i) # xmm0 to xmm7 + # + if IS_X86_32: + mc.LEA_rb(eax.value, +8) + mc.PUSH_r(eax.value) + elif IS_X86_64: + mc.LEA_rb(edi.value, +16) + mc.AND_ri(esp.value, -16) + # + f = llhelper(self._STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + addr = rffi.cast(lltype.Signed, f) + mc.CALL(imm(addr)) + # + mc.MOV(eax, heap(self.cpu.pos_exception())) + mc.TEST_rr(eax.value, eax.value) + mc.J_il8(rx86.Conditions['NZ'], 0) + jnz_location = mc.get_relative_pos() + # + if IS_X86_64: + # restore the registers + for i in range(7, -1, -1): + mc.MOVSD_xs(i, 8*i) + for i, reg in [(6, r9), (5, r8), (4, ecx), + (3, edx), (2, esi), (1, edi)]: + mc.MOV_rb(reg.value, -8*i) + # + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.RET() + # + # patch the JNZ above + offset = mc.get_relative_pos() - jnz_location + assert 0 < offset <= 127 + mc.overwrite(jnz_location-1, chr(offset)) + # clear the exception from the global position + mc.MOV(eax, heap(self.cpu.pos_exc_value())) + mc.MOV(heap(self.cpu.pos_exception()), imm0) + mc.MOV(heap(self.cpu.pos_exc_value()), imm0) + # save the current exception instance into fail_boxes_ptr[0] + adr = self.fail_boxes_ptr.get_addr_for_num(0) + mc.MOV(heap(adr), eax) + # call the helper function to set the GC flag on the fail_boxes_ptr + # array (note that there is no exception any more here) + addr = self.cpu.get_on_leave_jitted_int(save_exception=False) + mc.CALL(imm(addr)) + # + assert self.cpu.exit_frame_with_exception_v >= 0 + mc.MOV_ri(eax.value, self.cpu.exit_frame_with_exception_v) + # + # footer -- note the ADD, which skips the return address of this + # function, and will instead return to the caller's caller. Note + # also that we completely ignore the saved arguments, because we + # are interrupting the function. + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.ADD_ri(esp.value, WORD) + mc.RET() + # + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.stack_check_slowpath_imm = imm(rawstart) + def assemble_loop(self, inputargs, operations, looptoken, log): '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) @@ -468,6 +546,24 @@ class Assembler386(object): for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) + def _call_header_with_stack_check(self): + startaddr, length, slowpathaddr = self.cpu.insert_stack_check() + if slowpathaddr == 0: + pass # no stack check (e.g. not translated) + else: + self.mc.MOV(eax, esp) # MOV eax, current + self.mc.SUB(eax, heap(startaddr)) # SUB eax, [startaddr] + self.mc.CMP(eax, imm(length)) # CMP eax, length + self.mc.J_il8(rx86.Conditions['B'], 0) # JB .skip + jb_location = self.mc.get_relative_pos() + self.mc.CALL(self.stack_check_slowpath_imm) # CALL slowpath + # patch the JB above # .skip: + offset = self.mc.get_relative_pos() - jb_location + assert 0 < offset <= 127 + self.mc.overwrite(jb_location-1, chr(offset)) + # + self._call_header() + def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -485,7 +581,7 @@ class Assembler386(object): # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] @@ -526,7 +622,7 @@ class Assembler386(object): unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -115,6 +115,7 @@ class AbstractLLCPU(AbstractCPU): self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = lambda: (0, 0, 0) def _setup_exception_handling_translated(self): @@ -138,9 +139,20 @@ class AbstractLLCPU(AbstractCPU): # in the assignment to self.saved_exc_value, as needed. self.saved_exc_value = exc_value + from pypy.rlib import rstack + STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def insert_stack_check(): + startaddr = rstack._stack_get_start_adr() + length = rstack._stack_get_length() + f = llhelper(STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + slowpathaddr = rffi.cast(lltype.Signed, f) + return startaddr, length, slowpathaddr + self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = insert_stack_check def _setup_on_leave_jitted_untranslated(self): # assume we don't need a backend leave in this case From commits-noreply at bitbucket.org Mon Dec 13 18:55:04 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 11:55:04 -0600 (CST) Subject: [pypy-svn] pypy commit e73c414c7186: Fix. Message-ID: <20101213175504.128161E12DA@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Armin Rigo # Date 1292260877 0 # Node ID e73c414c718679a911fb725458acbc50b3d35640 # Parent 5e8eb0237e7f098cfa76aa48cbc06aa44da7fceb Fix. --- a/pypy/translator/goal/app_main.py +++ b/pypy/translator/goal/app_main.py @@ -207,10 +207,7 @@ def get_library_path(executable): break # found! return newpath -def setup_initial_paths(executable, nanos, ignore_environment=False, **extra): - # a substituted os if we are translated - global os - os = nanos +def setup_initial_paths(executable, ignore_environment=False, **extra): # find the full path to the executable, assuming that if there is no '/' # in the provided one then we must look along the $PATH if we_are_translated() and IS_WINDOWS and not executable.lower().endswith('.exe'): @@ -578,12 +575,15 @@ def print_banner(): '"license" for more information.') def entry_point(executable, argv, nanos): + # a substituted os if we are translated + global os + os = nanos try: cmdline = parse_command_line(argv) except CommandLineError, e: print_error(str(e)) return 2 - setup_initial_paths(executable, nanos, **cmdline) + setup_initial_paths(executable, **cmdline) return run_command_line(**cmdline) From commits-noreply at bitbucket.org Mon Dec 13 18:55:03 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 11:55:03 -0600 (CST) Subject: [pypy-svn] pypy commit e407bdf735c5: Progress on out-of-line guards. Use weakref for storing affected codes Message-ID: <20101213175503.7C1E11E1276@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Maciej Fijalkowski # Date 1292260525 0 # Node ID e407bdf735c5d1d48f6aef319f0822b0dc8a1e0a # Parent 1325866a95fd9e1636d13bc24b292a7e94eefa71 Progress on out-of-line guards. Use weakref for storing affected codes and cheat (horribly) to pretend that LoopToken is a low-level pointer --- a/pypy/jit/metainterp/history.py +++ b/pypy/jit/metainterp/history.py @@ -725,6 +725,20 @@ class LoopToken(AbstractDescr): was compiled; but the LoopDescr remains alive and points to the generated assembler. """ + + _TYPE = lltype.Ptr(lltype.GcStruct('dummy struct for tests')) + + def _normalizedcontainer(self): + return self + + def _getobj(self, check=True): + return self + _as_ptr = _getobj + _obj0 = None + _obj = property(_getobj) + def _was_freed(self): + return False + terminating = False # see TerminatingLoopToken in compile.py outermost_jitdriver_sd = None # specnodes = ... --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -490,6 +490,13 @@ class LLtypeCPU(BaseCPU): def get_invalidate_asm(self, TP, fieldname): def invalidate_asm(arg, fieldname): + prev = getattr(arg, fieldname) + next = prev + while next: + prev = next + x = llmemory.weakref_deref(history.LoopToken._TYPE, + prev.address) + next = next.next XXX # write me return invalidate_asm --- a/pypy/rpython/lltypesystem/rclass.py +++ b/pypy/rpython/lltypesystem/rclass.py @@ -87,7 +87,7 @@ LLFLAVOR = {'gc' : 'gc', # a linked-list of assembler codes to invalidate in case jit_invariant_fields # are modified ASMCODE = lltype.GcForwardReference() -ASMCODE.become(GcStruct('asmcode', ('address', llmemory.GCREF), +ASMCODE.become(GcStruct('asmcode', ('address', llmemory.WeakRefPtr), ('next', lltype.Ptr(ASMCODE)))) def cast_vtable_to_typeptr(vtable): --- a/pypy/jit/codewriter/jtransform.py +++ b/pypy/jit/codewriter/jtransform.py @@ -605,7 +605,8 @@ class Transformer(object): next = next.next prev.next = new_asmcode - args_s = [lltype_to_annotation(llmemory.GCREF)] * 2 + args_s = [lltype_to_annotation(llmemory.GCREF), + lltype_to_annotation(llmemory.WeakRefPtr)] s_result = lltype_to_annotation(lltype.Void) mixlevelann = MixLevelHelperAnnotator(self.cpu.rtyper) c_appender = mixlevelann.constfunc(appender, args_s, s_result) --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1,6 +1,5 @@ import py, os, sys from pypy.rpython.lltypesystem import lltype, llmemory, rclass -from pypy.rpython.lltypesystem.rclass import ASMCODE from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -2238,13 +2237,14 @@ class MetaInterp(object): op = op.copy_and_change(rop.CALL_ASSEMBLER, args=args, descr=token) self.history.operations.append(op) - def remember_jit_invariants(self, token): - lltoken = lltype.cast_opaque_ptr(llmemory.GCREF, token) + def remember_jit_invariants(self, loop): + lltoken_weakref = llmemory.weakref_create(loop.token) seen = {} for b_struct, c_appender in self.invariant_structs: if (b_struct, c_appender) not in seen: - heaptracker.int2adr(c_func.value).ptr(b_struct.value, lltoken) - seend[(b_struct, c_appender)] = None + heaptracker.int2adr(c_appender.value).ptr(b_struct.value, + lltoken_weakref) + seen[(b_struct, c_appender)] = None # ____________________________________________________________ From commits-noreply at bitbucket.org Mon Dec 13 19:01:42 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Mon, 13 Dec 2010 12:01:42 -0600 (CST) Subject: [pypy-svn] pypy commit e5b73981fc8d: merge heads Message-ID: <20101213180142.C239B6C12BB@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292263273 -3600 # Node ID e5b73981fc8d5cd14074426b0e439f0716379a31 # Parent 74bbe61137d921345adafcb80d004ce7d8860ace # Parent 85b74c8de1c96027c19741ddac3fb967390d7235 merge heads From david at codespeak.net Mon Dec 13 20:02:51 2010 From: david at codespeak.net (david at codespeak.net) Date: Mon, 13 Dec 2010 20:02:51 +0100 (CET) Subject: [pypy-svn] r80043 - in pypy/branch/arm-backend/pypy/jit/backend/arm: . helper Message-ID: <20101213190251.C0AC650813@codespeak.net> Author: david Date: Mon Dec 13 20:02:48 2010 New Revision: 80043 Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Log: Next register allocation fix Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/helper/regalloc.py Mon Dec 13 20:02:48 2010 @@ -49,13 +49,10 @@ arg2 = self.make_sure_var_in_reg(a1, selected_reg=r.r1) assert arg1 == r.r0 assert arg2 == r.r1 - spilled = False if isinstance(a0, Box) and self.stays_alive(a0): - spilled = True self.force_spill_var(a0) + self.possibly_free_var(a0) self.after_call(op.result) - if spilled: - self.possibly_free_var(a0) self.possibly_free_var(a1) self.possibly_free_var(op.result) return [] Modified: pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py ============================================================================== --- pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py (original) +++ pypy/branch/arm-backend/pypy/jit/backend/arm/regalloc.py Mon Dec 13 20:02:48 2010 @@ -143,8 +143,8 @@ boxes.append(box) l1, box = self._ensure_value_is_boxed(a1, [box]) boxes.append(box) - res = self.force_allocate_reg(op.result, boxes) self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) self.possibly_free_var(op.result) return [l0, l1, res] @@ -167,8 +167,8 @@ boxes.append(box) l1, box = self._ensure_value_is_boxed(a1, boxes) boxes.append(box) - res = self.force_allocate_reg(op.result, boxes) self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) self.possibly_free_var(op.result) return [l0, l1, res] @@ -181,8 +181,8 @@ reg2, box = self._ensure_value_is_boxed(a1, forbidden_vars=boxes) boxes.append(box) - res = self.force_allocate_reg(op.result, boxes) self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) self.possibly_free_var(op.result) return [reg1, reg2, res] @@ -241,8 +241,9 @@ def prepare_op_int_neg(self, op, fcond): l0, box = self._ensure_value_is_boxed(op.getarg(0)) - resloc = self.force_allocate_reg(op.result, [box]) - self.possibly_free_vars([box, op.result]) + self.possibly_free_var(box) + resloc = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) return [l0, resloc] prepare_op_int_invert = prepare_op_int_neg @@ -376,7 +377,7 @@ base_loc, base_box = self._ensure_value_is_boxed(a0) self.possibly_free_var(a0) self.possibly_free_var(base_box) - res = self.force_allocate_reg(op.result, [a0]) + res = self.force_allocate_reg(op.result) self.possibly_free_var(op.result) return [base_loc, imm(ofs), res, imm(size)] From hakanardo at codespeak.net Mon Dec 13 20:53:52 2010 From: hakanardo at codespeak.net (hakanardo at codespeak.net) Date: Mon, 13 Dec 2010 20:53:52 +0100 (CET) Subject: [pypy-svn] r80044 - in pypy/branch/jit-unroll-loops/pypy: jit/metainterp jit/metainterp/test module/pypyjit/test Message-ID: <20101213195352.D3E4350813@codespeak.net> Author: hakanardo Date: Mon Dec 13 20:53:50 2010 New Revision: 80044 Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_memmgr.py pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizebasic.py pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_pypy_c.py Log: fixed tests Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/compile.py Mon Dec 13 20:53:50 2010 @@ -113,12 +113,12 @@ if loop.preamble.operations is not None: send_loop_to_backend(metainterp_sd, loop, "loop") record_loop_or_bridge(loop) + token = loop.preamble.token if full_preamble_needed or not loop.preamble.token.short_preamble: send_loop_to_backend(metainterp_sd, loop.preamble, "entry bridge") insert_loop_token(old_loop_tokens, loop.preamble.token) jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( greenkey, loop.preamble.token) - token = loop.preamble.token record_loop_or_bridge(loop.preamble) return token else: Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_memmgr.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_memmgr.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_memmgr.py Mon Dec 13 20:53:50 2010 @@ -132,7 +132,7 @@ res = self.meta_interp(f, [], loop_longevity=1) assert res == 42 # we should see a loop for each call to g() - self.check_tree_loop_count(8 + 20*2) + self.check_tree_loop_count(8 + 20*2*2) def test_throw_away_old_loops(self): myjitdriver = JitDriver(greens=['m'], reds=['n']) @@ -157,7 +157,7 @@ res = self.meta_interp(f, [], loop_longevity=3) assert res == 42 - self.check_tree_loop_count(2 + 10*4) # 42 :-) + self.check_tree_loop_count(2 + 10*4*2) def test_call_assembler_keep_alive(self): myjitdriver1 = JitDriver(greens=['m'], reds=['n']) @@ -191,7 +191,7 @@ res = self.meta_interp(f, [1], loop_longevity=4, inline=True) assert res == 42 - self.check_tree_loop_count(8) + self.check_tree_loop_count(12) # ____________________________________________________________ Modified: pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizebasic.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizebasic.py (original) +++ pypy/branch/jit-unroll-loops/pypy/jit/metainterp/test/test_optimizebasic.py Mon Dec 13 20:53:50 2010 @@ -262,12 +262,14 @@ OptString, OptHeap, Optimizer) + from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall + optimizations = [OptIntBounds(), OptRewrite(), OptVirtualize(), OptString(), OptHeap(), - #OptFfiCall(), + OptFfiCall(), ] optimizer = Optimizer(metainterp_sd, loop, optimizations) optimizer.propagate_all_forward() Modified: pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_pypy_c.py ============================================================================== --- pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_pypy_c.py (original) +++ pypy/branch/jit-unroll-loops/pypy/module/pypyjit/test/test_pypy_c.py Mon Dec 13 20:53:50 2010 @@ -422,7 +422,7 @@ ([1000], 49500), ([10000], 495000), ([100000], 4950000)) - assert len(self.loops) == 3 + assert len(self.rawloops) + len(self.rawentrybridges) == 4 op, = self.get_by_bytecode("CALL_FUNCTION_KW") # XXX a bit too many guards, but better than before assert len(op.get_opnames("guard")) <= 12 From antocuni at codespeak.net Mon Dec 13 23:54:45 2010 From: antocuni at codespeak.net (antocuni at codespeak.net) Date: Mon, 13 Dec 2010 23:54:45 +0100 (CET) Subject: [pypy-svn] r80045 - pypy/extradoc/planning/hg-migration Message-ID: <20101213225445.0176F50813@codespeak.net> Author: antocuni Date: Mon Dec 13 23:54:43 2010 New Revision: 80045 Added: pypy/extradoc/planning/hg-migration/active-branches-howto.txt - copied, changed from r80044, pypy/extradoc/planning/hg-migration/merge-howto.txt Removed: pypy/extradoc/planning/hg-migration/merge-howto.txt Log: rename the file, and add a paragraph for how to import svn externals Copied: pypy/extradoc/planning/hg-migration/active-branches-howto.txt (from r80044, pypy/extradoc/planning/hg-migration/merge-howto.txt) ============================================================================== --- pypy/extradoc/planning/hg-migration/merge-howto.txt (original) +++ pypy/extradoc/planning/hg-migration/active-branches-howto.txt Mon Dec 13 23:54:43 2010 @@ -1,7 +1,32 @@ .. -*- mode: rst -*- +Complete the migration of active branches +========================================== + +Now that the SVN repo has been migrate to mercurial, there are two small steps +that have to be done manually if you want to continue the devlopment of a +branch: + + 1. import the svn externals + + 2. tell mercurial the last revision where you "merged from trunk", to avoid + false conflicts during the merge + +Import the svn externals +------------------------- + +This is easy. There is already a changeset that contains all the needed +externals (as subrepo), you just need to merge it into your branch:: + + $ hg up the-name-of-your-branch + + $ hg merge -r bf2c629d0071 + + $ hg ci -m 'import svn externals into the branch' + + How to merge existing SVN branches in mercurial -================================================ +------------------------------------------------ This document applies to branches that: From afa at codespeak.net Tue Dec 14 00:05:08 2010 From: afa at codespeak.net (afa at codespeak.net) Date: Tue, 14 Dec 2010 00:05:08 +0100 (CET) Subject: [pypy-svn] r80046 - in pypy/branch/fast-forward: . pypy/interpreter pypy/jit/backend pypy/jit/backend/llsupport pypy/jit/backend/llsupport/test pypy/jit/backend/x86 pypy/jit/backend/x86/test pypy/jit/backend/x86/tool pypy/jit/metainterp pypy/module/posix pypy/module/posix/test pypy/module/sys pypy/objspace/std pypy/objspace/std/test pypy/rlib pypy/rpython/lltypesystem pypy/rpython/module pypy/translator/c/src pypy/translator/c/test Message-ID: <20101213230508.C8C5F50813@codespeak.net> Author: afa Date: Tue Dec 14 00:05:06 2010 New Revision: 80046 Added: pypy/branch/fast-forward/.hgignore - copied unchanged from r80045, pypy/trunk/.hgignore Modified: pypy/branch/fast-forward/pypy/interpreter/argument.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py pypy/branch/fast-forward/pypy/jit/backend/model.py pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py pypy/branch/fast-forward/pypy/jit/backend/x86/codebuf.py pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py pypy/branch/fast-forward/pypy/jit/backend/x86/rx86.py pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_ztranslation.py pypy/branch/fast-forward/pypy/jit/backend/x86/tool/viewcode.py pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py pypy/branch/fast-forward/pypy/module/posix/__init__.py pypy/branch/fast-forward/pypy/module/posix/interp_posix.py pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py pypy/branch/fast-forward/pypy/module/sys/__init__.py pypy/branch/fast-forward/pypy/module/sys/version.py pypy/branch/fast-forward/pypy/objspace/std/dictmultiobject.py pypy/branch/fast-forward/pypy/objspace/std/test/test_dictmultiobject.py pypy/branch/fast-forward/pypy/rlib/rposix.py pypy/branch/fast-forward/pypy/rlib/rstack.py pypy/branch/fast-forward/pypy/rpython/lltypesystem/lloperation.py pypy/branch/fast-forward/pypy/rpython/module/ll_os.py pypy/branch/fast-forward/pypy/translator/c/src/stack.h pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py Log: svn merge -r80002:80045 ../trunk Modified: pypy/branch/fast-forward/pypy/interpreter/argument.py ============================================================================== --- pypy/branch/fast-forward/pypy/interpreter/argument.py (original) +++ pypy/branch/fast-forward/pypy/interpreter/argument.py Tue Dec 14 00:05:06 2010 @@ -103,9 +103,9 @@ make_sure_not_resized(self.keywords_w) make_sure_not_resized(self.arguments_w) - if w_stararg is not None and space.is_true(w_stararg): + if w_stararg is not None: self._combine_starargs_wrapped(w_stararg) - if w_starstararg is not None and space.is_true(w_starstararg): + if w_starstararg is not None: self._combine_starstarargs_wrapped(w_starstararg) # if we have a call where **args are used at the callsite # we shouldn't let the JIT see the argument matching Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/asmmemmgr.py Tue Dec 14 00:05:06 2010 @@ -2,6 +2,8 @@ from pypy.rlib.rarithmetic import intmask, r_uint, LONG_BIT from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rmmap +from pypy.rlib.debug import debug_start, debug_print, debug_stop +from pypy.rlib.debug import have_debug_prints from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -265,6 +267,31 @@ targetindex -= self.SUBBLOCK_SIZE assert not block + def _dump(self, addr, logname, backend=None): + debug_start(logname) + if have_debug_prints(): + # + if backend is not None: + debug_print('BACKEND', backend) + # + from pypy.jit.backend.hlinfo import highleveljitinfo + if highleveljitinfo.sys_executable: + debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable) + # + HEX = '0123456789ABCDEF' + dump = [] + src = rffi.cast(rffi.CCHARP, addr) + for p in range(self.get_relative_pos()): + o = ord(src[p]) + dump.append(HEX[o >> 4]) + dump.append(HEX[o & 15]) + debug_print('CODE_DUMP', + '@%x' % addr, + '+0 ', # backwards compatibility + ''.join(dump)) + # + debug_stop(logname) + def materialize(self, asmmemmgr, allblocks, gcrootmap=None): size = self.get_relative_pos() malloced = asmmemmgr.malloc(size, size) Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/llmodel.py Tue Dec 14 00:05:06 2010 @@ -115,6 +115,7 @@ self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = lambda: (0, 0, 0) def _setup_exception_handling_translated(self): @@ -138,9 +139,20 @@ # in the assignment to self.saved_exc_value, as needed. self.saved_exc_value = exc_value + from pypy.rlib import rstack + STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def insert_stack_check(): + startaddr = rstack._stack_get_start_adr() + length = rstack._stack_get_length() + f = llhelper(STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + slowpathaddr = rffi.cast(lltype.Signed, f) + return startaddr, length, slowpathaddr + self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = insert_stack_check def _setup_on_leave_jitted_untranslated(self): # assume we don't need a backend leave in this case Modified: pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/llsupport/test/test_asmmemmgr.py Tue Dec 14 00:05:06 2010 @@ -3,6 +3,7 @@ from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib import debug def test_get_index(): @@ -184,9 +185,11 @@ def test_blockbuildermixin(translated=True): mc = BlockBuilderMixin(translated) + writtencode = [] for i in range(mc.SUBBLOCK_SIZE * 2 + 3): assert mc.get_relative_pos() == i mc.writechar(chr(i % 255)) + writtencode.append(chr(i % 255)) if translated: assert mc._cursubindex == 3 assert mc._cursubblock @@ -196,16 +199,26 @@ # for i in range(0, mc.SUBBLOCK_SIZE * 2 + 3, 2): mc.overwrite(i, chr((i + 63) % 255)) + writtencode[i] = chr((i + 63) % 255) # p = lltype.malloc(rffi.CCHARP.TO, mc.SUBBLOCK_SIZE * 2 + 3, flavor='raw') addr = rffi.cast(lltype.Signed, p) mc.copy_to_raw_memory(addr) # for i in range(mc.SUBBLOCK_SIZE * 2 + 3): - if i & 1: - assert p[i] == chr(i % 255) - else: - assert p[i] == chr((i + 63) % 255) + assert p[i] == writtencode[i] + # + debug._log = debug.DebugLog() + try: + mc._dump(addr, 'test-logname-section') + log = list(debug._log) + finally: + debug._log = None + encoded = ''.join(writtencode).encode('hex').upper() + ataddr = '@%x' % addr + assert log == [('test-logname-section', + [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])] + # lltype.free(p, flavor='raw') def test_blockbuildermixin2(): Modified: pypy/branch/fast-forward/pypy/jit/backend/model.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/model.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/model.py Tue Dec 14 00:05:06 2010 @@ -8,6 +8,7 @@ done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + exit_frame_with_exception_v = -1 total_compiled_loops = 0 total_compiled_bridges = 0 total_freed_loops = 0 Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/assembler.py Tue Dec 14 00:05:06 2010 @@ -84,6 +84,7 @@ self.fail_boxes_count = 0 self._current_depths_cache = (0, 0) self.datablockwrapper = None + self.stack_check_slowpath_imm = imm0 self.teardown() def leave_jitted_hook(self): @@ -122,6 +123,7 @@ self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + self._build_stack_check_slowpath() debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') @@ -194,6 +196,82 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.malloc_fixedsize_slowpath2 = rawstart + _STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def _build_stack_check_slowpath(self): + from pypy.rlib import rstack + mc = codebuf.MachineCodeBlockWrapper() + mc.PUSH_r(ebp.value) + mc.MOV_rr(ebp.value, esp.value) + # + if IS_X86_64: + # on the x86_64, we have to save all the registers that may + # have been used to pass arguments + for reg in [edi, esi, edx, ecx, r8, r9]: + mc.PUSH_r(reg.value) + mc.SUB_ri(esp.value, 8*8) + for i in range(8): + mc.MOVSD_sx(8*i, i) # xmm0 to xmm7 + # + if IS_X86_32: + mc.LEA_rb(eax.value, +8) + mc.PUSH_r(eax.value) + elif IS_X86_64: + mc.LEA_rb(edi.value, +16) + mc.AND_ri(esp.value, -16) + # + f = llhelper(self._STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + addr = rffi.cast(lltype.Signed, f) + mc.CALL(imm(addr)) + # + mc.MOV(eax, heap(self.cpu.pos_exception())) + mc.TEST_rr(eax.value, eax.value) + mc.J_il8(rx86.Conditions['NZ'], 0) + jnz_location = mc.get_relative_pos() + # + if IS_X86_64: + # restore the registers + for i in range(7, -1, -1): + mc.MOVSD_xs(i, 8*i) + for i, reg in [(6, r9), (5, r8), (4, ecx), + (3, edx), (2, esi), (1, edi)]: + mc.MOV_rb(reg.value, -8*i) + # + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.RET() + # + # patch the JNZ above + offset = mc.get_relative_pos() - jnz_location + assert 0 < offset <= 127 + mc.overwrite(jnz_location-1, chr(offset)) + # clear the exception from the global position + mc.MOV(eax, heap(self.cpu.pos_exc_value())) + mc.MOV(heap(self.cpu.pos_exception()), imm0) + mc.MOV(heap(self.cpu.pos_exc_value()), imm0) + # save the current exception instance into fail_boxes_ptr[0] + adr = self.fail_boxes_ptr.get_addr_for_num(0) + mc.MOV(heap(adr), eax) + # call the helper function to set the GC flag on the fail_boxes_ptr + # array (note that there is no exception any more here) + addr = self.cpu.get_on_leave_jitted_int(save_exception=False) + mc.CALL(imm(addr)) + # + assert self.cpu.exit_frame_with_exception_v >= 0 + mc.MOV_ri(eax.value, self.cpu.exit_frame_with_exception_v) + # + # footer -- note the ADD, which skips the return address of this + # function, and will instead return to the caller's caller. Note + # also that we completely ignore the saved arguments, because we + # are interrupting the function. + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.ADD_ri(esp.value, WORD) + mc.RET() + # + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.stack_check_slowpath_imm = imm(rawstart) + def assemble_loop(self, inputargs, operations, looptoken, log): '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) @@ -468,6 +546,24 @@ for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) + def _call_header_with_stack_check(self): + startaddr, length, slowpathaddr = self.cpu.insert_stack_check() + if slowpathaddr == 0: + pass # no stack check (e.g. not translated) + else: + self.mc.MOV(eax, esp) # MOV eax, current + self.mc.SUB(eax, heap(startaddr)) # SUB eax, [startaddr] + self.mc.CMP(eax, imm(length)) # CMP eax, length + self.mc.J_il8(rx86.Conditions['B'], 0) # JB .skip + jb_location = self.mc.get_relative_pos() + self.mc.CALL(self.stack_check_slowpath_imm) # CALL slowpath + # patch the JB above # .skip: + offset = self.mc.get_relative_pos() - jb_location + assert 0 < offset <= 127 + self.mc.overwrite(jb_location-1, chr(offset)) + # + self._call_header() + def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -485,7 +581,7 @@ # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] @@ -526,7 +622,7 @@ unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/codebuf.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/codebuf.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/codebuf.py Tue Dec 14 00:05:06 2010 @@ -10,8 +10,10 @@ # like this if IS_X86_32: codebuilder_cls = X86_32_CodeBuilder + backend_name = 'x86' elif IS_X86_64: codebuilder_cls = X86_64_CodeBuilder + backend_name = 'x86_64' class MachineCodeBlockWrapper(BlockBuilderMixin, @@ -34,3 +36,4 @@ adr = rffi.cast(rffi.LONGP, p - WORD) adr[0] = intmask(adr[0] - p) valgrind.discard_translations(addr, self.get_relative_pos()) + self._dump(addr, "jit-backend-dump", backend_name) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/regalloc.py Tue Dec 14 00:05:06 2010 @@ -101,7 +101,6 @@ return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): - exc = False def __init__(self, assembler, translate_support_code=False): assert isinstance(translate_support_code, bool) @@ -428,7 +427,10 @@ locs = [self.loc(op.getarg(i)) for i in range(op.numargs())] locs_are_ref = [op.getarg(i).type == REF for i in range(op.numargs())] fail_index = self.assembler.cpu.get_fail_descr_number(op.getdescr()) - self.assembler.generate_failure(fail_index, locs, self.exc, + # note: no exception should currently be set in llop.get_exception_addr + # even if this finish may be an exit_frame_with_exception (in this case + # the exception instance is in locs[0]). + self.assembler.generate_failure(fail_index, locs, False, locs_are_ref) self.possibly_free_vars_for_op(op) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/rx86.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/rx86.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/rx86.py Tue Dec 14 00:05:06 2010 @@ -347,6 +347,8 @@ INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') INSN_br = insn(rex_w, chr(base+1), register(2,8), stack_bp(1)) INSN_rb = insn(rex_w, chr(base+3), register(1,8), stack_bp(2)) + INSN_rm = insn(rex_w, chr(base+3), register(1,8), mem_reg_plus_const(2)) + INSN_rj = insn(rex_w, chr(base+3), register(1,8), '\x05', immediate(2)) INSN_bi8 = insn(rex_w, '\x83', orbyte(base), stack_bp(1), immediate(2,'b')) INSN_bi32= insn(rex_w, '\x81', orbyte(base), stack_bp(1), immediate(2)) @@ -364,7 +366,7 @@ INSN_bi32(mc, offset, immed) INSN_bi._always_inline_ = True # try to constant-fold single_byte() - return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br + return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br, INSN_rm, INSN_rj def select_8_or_32_bit_immed(insn_8, insn_32): def INSN(*args): @@ -442,24 +444,22 @@ # ------------------------------ Arithmetic ------------------------------ - ADD_ri, ADD_rr, ADD_rb, _, _ = common_modes(0) - OR_ri, OR_rr, OR_rb, _, _ = common_modes(1) - AND_ri, AND_rr, AND_rb, _, _ = common_modes(4) - SUB_ri, SUB_rr, SUB_rb, _, _ = common_modes(5) - XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) - CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) + ADD_ri, ADD_rr, ADD_rb, _, _, ADD_rm, ADD_rj = common_modes(0) + OR_ri, OR_rr, OR_rb, _, _, OR_rm, OR_rj = common_modes(1) + AND_ri, AND_rr, AND_rb, _, _, AND_rm, AND_rj = common_modes(4) + SUB_ri, SUB_rr, SUB_rb, _, _, SUB_rm, SUB_rj = common_modes(5) + XOR_ri, XOR_rr, XOR_rb, _, _, XOR_rm, XOR_rj = common_modes(6) + CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br, CMP_rm, CMP_rj = common_modes(7) CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32) - CMP_rm = insn(rex_w, '\x3B', register(1, 8), mem_reg_plus_const(2)) CMP_mr = insn(rex_w, '\x39', register(2, 8), mem_reg_plus_const(1)) CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) - CMP_rj = insn(rex_w, '\x3B', register(1, 8), '\x05', immediate(2)) CMP32_mi = insn(rex_nw, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_ztranslation.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_ztranslation.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/test/test_ztranslation.py Tue Dec 14 00:05:06 2010 @@ -93,6 +93,9 @@ assert res == expected def test_direct_assembler_call_translates(self): + """Test CALL_ASSEMBLER and the recursion limit""" + from pypy.rlib.rstackovf import StackOverflow + class Thing(object): def __init__(self, val): self.val = val @@ -135,9 +138,35 @@ i += 1 return frame.thing.val - res = self.meta_interp(main, [0], inline=True, + driver2 = JitDriver(greens = [], reds = ['n']) + + def main2(bound): + try: + while portal2(bound) == -bound+1: + bound *= 2 + except StackOverflow: + pass + return bound + + def portal2(n): + while True: + driver2.jit_merge_point(n=n) + n -= 1 + if n <= 0: + return n + n = portal2(n) + assert portal2(10) == -9 + + def mainall(codeno, bound): + return main(codeno) + main2(bound) + + res = self.meta_interp(mainall, [0, 1], inline=True, policy=StopAtXPolicy(change)) - assert res == main(0) + print hex(res) + assert res & 255 == main(0) + bound = res & ~255 + assert 1024 <= bound <= 131072 + assert bound & (bound-1) == 0 # a power of two class TestTranslationRemoveTypePtrX86(CCompiledMixin): Modified: pypy/branch/fast-forward/pypy/jit/backend/x86/tool/viewcode.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/backend/x86/tool/viewcode.py (original) +++ pypy/branch/fast-forward/pypy/jit/backend/x86/tool/viewcode.py Tue Dec 14 00:05:06 2010 @@ -1,11 +1,11 @@ #! /usr/bin/env python """ -Viewer for the CODE_DUMP output of compiled programs generating code. +Viewer for the output of compiled programs generating code. +Use on the log files created with 'PYPYLOG=jit-backend-dump:log'. Try: - ./viewcode.py dumpfile.txt -or - /tmp/usession-xxx/testing_1/testing_1 -var 4 2>&1 | ./viewcode.py + ./viewcode.py --text log # text only disassembly + ./viewcode.py log # also includes a pygame viewer """ import autopath @@ -179,6 +179,7 @@ self.symbols = {} self.logentries = {} self.backend_name = None + self.executable_name = None def parse(self, f, textonly=True): for line in f: @@ -214,7 +215,9 @@ self.logentries[addr] = pieces[3] elif line.startswith('SYS_EXECUTABLE '): filename = line[len('SYS_EXECUTABLE '):].strip() - self.symbols.update(load_symbols(filename)) + if filename != self.executable_name: + self.symbols.update(load_symbols(filename)) + self.executable_name = filename def find_cross_references(self): # find cross-references between blocks @@ -375,10 +378,19 @@ showgraph = False else: showgraph = True - if len(sys.argv) == 1: - f = sys.stdin - else: - f = open(sys.argv[1], 'r') + if len(sys.argv) != 2: + print >> sys.stderr, __doc__ + sys.exit(2) + # + import cStringIO + from pypy.tool import logparser + log1 = logparser.parse_log_file(sys.argv[1]) + text1 = logparser.extract_category(log1, catprefix='jit-backend-dump') + f = cStringIO.StringIO() + f.writelines(text1) + f.seek(0) + del log1, text1 + # world = World() world.parse(f) if showgraph: Modified: pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/fast-forward/pypy/jit/metainterp/pyjitpl.py Tue Dec 14 00:05:06 2010 @@ -1294,6 +1294,10 @@ num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) # + tokens = self.loop_tokens_exit_frame_with_exception_ref + num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) + self.cpu.exit_frame_with_exception_v = num + # self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): Modified: pypy/branch/fast-forward/pypy/module/posix/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/__init__.py Tue Dec 14 00:05:06 2010 @@ -71,6 +71,8 @@ if hasattr(os, 'chown'): interpleveldefs['chown'] = 'interp_posix.chown' + if hasattr(os, 'lchown'): + interpleveldefs['lchown'] = 'interp_posix.lchown' if hasattr(os, 'ftruncate'): interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' if hasattr(os, 'fsync'): @@ -86,6 +88,8 @@ if hasattr(os, 'kill') and sys.platform != 'win32': interpleveldefs['kill'] = 'interp_posix.kill' interpleveldefs['abort'] = 'interp_posix.abort' + if hasattr(os, 'killpg'): + interpleveldefs['killpg'] = 'interp_posix.killpg' if hasattr(os, 'getpid'): interpleveldefs['getpid'] = 'interp_posix.getpid' if hasattr(os, 'link'): @@ -117,6 +121,12 @@ interpleveldefs['ttyname'] = 'interp_posix.ttyname' if hasattr(os, 'getloadavg'): interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' + if hasattr(os, 'mkfifo'): + interpleveldefs['mkfifo'] = 'interp_posix.mkfifo' + if hasattr(os, 'mknod'): + interpleveldefs['mknod'] = 'interp_posix.mknod' + if hasattr(os, 'nice'): + interpleveldefs['nice'] = 'interp_posix.nice' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', Modified: pypy/branch/fast-forward/pypy/module/posix/interp_posix.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/interp_posix.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/interp_posix.py Tue Dec 14 00:05:06 2010 @@ -161,6 +161,7 @@ ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] def fsync(space, w_fd): + """Force write of file with filedescriptor to disk.""" fd = space.c_filedescriptor_w(w_fd) try: os.fsync(fd) @@ -169,6 +170,8 @@ fsync.unwrap_spec = [ObjSpace, W_Root] def fdatasync(space, w_fd): + """Force write of file with filedescriptor to disk. +Does not force update of metadata.""" fd = space.c_filedescriptor_w(w_fd) try: os.fdatasync(fd) @@ -177,6 +180,8 @@ fdatasync.unwrap_spec = [ObjSpace, W_Root] def fchdir(space, w_fd): + """Change to the directory of the given file descriptor. fildes must be +opened on a directory, not a file.""" fd = space.c_filedescriptor_w(w_fd) try: os.fchdir(fd) @@ -557,6 +562,27 @@ raise wrap_oserror(space, e) rename.unwrap_spec = [ObjSpace, W_Root, W_Root] +def mkfifo(space, w_filename, mode=0666): + """Create a FIFO (a POSIX named pipe).""" + try: + dispatch_filename(rposix.mkfifo)(space, w_filename, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_filename) +mkfifo.unwrap_spec = [ObjSpace, W_Root, "c_int"] + +def mknod(space, w_filename, mode=0600, device=0): + """Create a filesystem node (file, device special file or named pipe) +named filename. mode specifies both the permissions to use and the +type of node to be created, being combined (bitwise OR) with one of +S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. For S_IFCHR and S_IFBLK, +device defines the newly created device special file (probably using +os.makedev()), otherwise it is ignored.""" + try: + dispatch_filename(rposix.mknod)(space, w_filename, mode, device) + except OSError, e: + raise wrap_oserror2(space, e, w_filename) +mknod.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] + def umask(space, mask): "Set the current numeric umask and return the previous umask." prevmask = os.umask(mask) @@ -580,6 +606,14 @@ raise wrap_oserror(space, e) kill.unwrap_spec = [ObjSpace, "c_int", "c_int"] +def killpg(space, pgid, sig): + "Kill a process group with a signal." + try: + os.killpg(pgid, sig) + except OSError, e: + raise wrap_oserror(space, e) +killpg.unwrap_spec = [ObjSpace, "c_int", "c_int"] + def abort(space): """Abort the interpreter immediately. This 'dumps core' or otherwise fails in the hardest way possible on the hosting operating system.""" @@ -1034,6 +1068,14 @@ return space.w_None chown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] +def lchown(space, path, uid, gid): + try: + os.lchown(path, uid, gid) + except OSError, e: + raise wrap_oserror(space, e, path) + return space.w_None +lchown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] + def getloadavg(space): try: load = os.getloadavg() @@ -1045,6 +1087,15 @@ space.wrap(load[2])]) getloadavg.unwrap_spec = [ObjSpace] +def nice(space, inc): + "Decrease the priority of process by inc and return the new priority." + try: + res = os.nice(inc) + except OSError, e: + raise wrap_oserror(space, e) + return space.wrap(res) +nice.unwrap_spec = [ObjSpace, "c_int"] + if _WIN: from pypy.rlib import rwin32 Modified: pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py (original) +++ pypy/branch/fast-forward/pypy/module/posix/test/test_posix2.py Tue Dec 14 00:05:06 2010 @@ -672,6 +672,67 @@ f.close() os.chown(self.path, os.getuid(), os.getgid()) + if hasattr(os, 'lchown'): + def test_lchown(self): + os = self.posix + os.unlink(self.path) + raises(OSError, os.lchown, self.path, os.getuid(), os.getgid()) + os.symlink('foobar', self.path) + os.lchown(self.path, os.getuid(), os.getgid()) + + if hasattr(os, 'mkfifo'): + def test_mkfifo(self): + os = self.posix + os.mkfifo(self.path2 + 'test_mkfifo', 0666) + st = os.lstat(self.path2 + 'test_mkfifo') + import stat + assert stat.S_ISFIFO(st.st_mode) + + if hasattr(os, 'mknod'): + def test_mknod(self): + import stat + os = self.posix + # not very useful: os.mknod() without specifying 'mode' + os.mknod(self.path2 + 'test_mknod-1') + st = os.lstat(self.path2 + 'test_mknod-1') + assert stat.S_ISREG(st.st_mode) + # os.mknod() with S_IFIFO + os.mknod(self.path2 + 'test_mknod-2', 0600 | stat.S_IFIFO) + st = os.lstat(self.path2 + 'test_mknod-2') + assert stat.S_ISFIFO(st.st_mode) + + def test_mknod_with_ifchr(self): + # os.mknod() with S_IFCHR + # -- usually requires root priviledges -- + os = self.posix + if hasattr(os.lstat('.'), 'st_rdev'): + import stat + try: + os.mknod(self.path2 + 'test_mknod-3', 0600 | stat.S_IFCHR, + 0x105) + except OSError, e: + skip("os.mknod() with S_IFCHR: got %r" % (e,)) + else: + st = os.lstat(self.path2 + 'test_mknod-3') + assert stat.S_ISCHR(st.st_mode) + assert st.st_rdev == 0x105 + + if hasattr(os, 'nice') and hasattr(os, 'fork') and hasattr(os, 'waitpid'): + def test_nice(self): + os = self.posix + myprio = os.nice(0) + # + pid = os.fork() + if pid == 0: # in the child + res = os.nice(3) + os._exit(res) + # + pid1, status1 = os.waitpid(pid, 0) + assert pid1 == pid + assert os.WIFEXITED(status1) + assert os.WEXITSTATUS(status1) == myprio + 3 + + class AppTestEnvironment(object): def setup_class(cls): cls.space = space Modified: pypy/branch/fast-forward/pypy/module/sys/__init__.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/__init__.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/__init__.py Tue Dec 14 00:05:06 2010 @@ -64,6 +64,7 @@ 'pypy_version_info' : 'version.get_pypy_version_info(space)', 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', + '_mercurial' : 'version.get_mercurial_info(space)', 'hexversion' : 'version.get_hexversion(space)', 'displayhook' : 'hook.displayhook', Modified: pypy/branch/fast-forward/pypy/module/sys/version.py ============================================================================== --- pypy/branch/fast-forward/pypy/module/sys/version.py (original) +++ pypy/branch/fast-forward/pypy/module/sys/version.py Tue Dec 14 00:05:06 2010 @@ -98,6 +98,39 @@ space.wrap(svnbranch), space.wrap(str(svn_revision()))]) +def get_mercurial_info(space): + '''Obtain Mercurial version information by invoking the 'hg' command.''' + # TODO: support extracting from .hg_archival.txt + import py + from subprocess import Popen, PIPE + + pypyroot = os.path.abspath(os.path.join(pypydir, '..')) + hgexe = py.path.local.sysfind('hg') + + if hgexe and os.path.isdir(os.path.join(pypyroot, '.hg')): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull + + p = Popen([str(hgexe), 'id', '-i', pypyroot], stdout=PIPE, env=env) + hgid = p.stdout.read().strip() + + p = Popen([str(hgexe), 'id', '-t', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip().split()[0] + + if hgtag == 'tip': + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip() + + return space.newtuple([space.wrap('PyPy'), + space.wrap(hgtag), + space.wrap(hgid)]) + else: + return space.w_None + def tuple2hex(ver): d = {'alpha': 0xA, 'beta': 0xB, Modified: pypy/branch/fast-forward/pypy/objspace/std/dictmultiobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/dictmultiobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/dictmultiobject.py Tue Dec 14 00:05:06 2010 @@ -56,9 +56,8 @@ else: if w_type is None: w_type = space.w_dict - w_self = space.allocate_instance(W_DictMultiObject, w_type) - W_DictMultiObject.__init__(w_self, space) - w_self.initialize_as_rdict() + w_self = space.allocate_instance(EmptyDictImplementation, w_type) + EmptyDictImplementation.__init__(w_self, space) return w_self def __init__(self, space): @@ -108,7 +107,7 @@ def impl_setitem_str(self, key, w_value): raise NotImplementedError("abstract base class") - def impl_setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): raise NotImplementedError("abstract base class") def impl_delitem(self, w_key): @@ -409,6 +408,45 @@ return self.shadowed[i] +class EmptyDictImplementation(W_DictMultiObject): + def __init__(self, space): + self.space = space + + def impl_setitem(self, w_key, w_value): + self._as_rdict().impl_fallback_setitem(w_key, w_value) + + def impl_setitem_str(self, key, w_value): + self._as_rdict().impl_fallback_setitem_str(key, w_value) + + def impl_delitem(self, w_key): + raise KeyError + + def impl_length(self): + return 0 + + def impl_getitem_str(self, key): + return None + + def impl_getitem(self, w_key): + # in case the key is unhashable, try to hash it + self.space.hash(w_key) + # return None anyway + return None + + def impl_iter(self): + # XXX I guess it's not important to be fast in this case? + return self._as_rdict().impl_fallback_iter() + + def impl_clear(self): + self.r_dict_content = None + + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() + return self + + def _clear_fields(self): + pass + class RDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) Modified: pypy/branch/fast-forward/pypy/objspace/std/test/test_dictmultiobject.py ============================================================================== --- pypy/branch/fast-forward/pypy/objspace/std/test/test_dictmultiobject.py (original) +++ pypy/branch/fast-forward/pypy/objspace/std/test/test_dictmultiobject.py Tue Dec 14 00:05:06 2010 @@ -16,6 +16,7 @@ space = self.space d = self.space.newdict() assert not self.space.is_true(d) + assert d.r_dict_content is None def test_nonempty(self): space = self.space Modified: pypy/branch/fast-forward/pypy/rlib/rposix.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rposix.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rposix.py Tue Dec 14 00:05:06 2010 @@ -135,6 +135,20 @@ else: return os.rmdir(path.as_bytes()) + at specialize.argtype(0) +def mkfifo(path, mode): + if isinstance(path, str): + os.mkfifo(path, mode) + else: + os.mkfifo(path.as_bytes(), mode) + + at specialize.argtype(0) +def mknod(path, mode, device): + if isinstance(path, str): + os.mknod(path, mode, device) + else: + os.mknod(path.as_bytes(), mode, device) + if os.name == 'nt': import nt def _getfullpathname(path): Modified: pypy/branch/fast-forward/pypy/rlib/rstack.py ============================================================================== --- pypy/branch/fast-forward/pypy/rlib/rstack.py (original) +++ pypy/branch/fast-forward/pypy/rlib/rstack.py Tue Dec 14 00:05:06 2010 @@ -6,21 +6,22 @@ import inspect from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rarithmetic import r_uint +from pypy.rlib import rgc from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.controllerentry import Controller, SomeControlledInstance from pypy.translator.tool.cbuild import ExternalCompilationInfo def stack_unwind(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_unwind(lltype.Void) raise RuntimeError("cannot unwind stack in non-translated versions") def stack_capture(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop ptr = llop.stack_capture(OPAQUE_STATE_HEADER_PTR) return frame_stack_top_controller.box(ptr) raise RuntimeError("cannot unwind stack in non-translated versions") @@ -28,26 +29,57 @@ def stack_frames_depth(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_frames_depth(lltype.Signed) else: return len(inspect.stack()) +# ____________________________________________________________ + compilation_info = ExternalCompilationInfo(includes=['src/stack.h']) -stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, - compilation_info=compilation_info, - _nowrapper=True, - _callable=lambda: _zero, - sandboxsafe=True) -_zero = rffi.cast(rffi.INT, 0) +def llexternal(name, args, res): + return rffi.llexternal(name, args, res, compilation_info=compilation_info, + sandboxsafe=True, _nowrapper=True) + +_stack_get_start = llexternal('LL_stack_get_start', [], lltype.Signed) +_stack_get_length = llexternal('LL_stack_get_length', [], lltype.Signed) +_stack_too_big_slowpath = llexternal('LL_stack_too_big_slowpath', + [lltype.Signed], lltype.Char) +# the following is used by the JIT +_stack_get_start_adr = llexternal('LL_stack_get_start_adr', [], lltype.Signed) + def stack_check(): - if rffi.cast(lltype.Signed, stack_too_big()): + if not we_are_translated(): + return + # + # Load the "current" stack position, or at least some address that + # points close to the current stack head + current = llop.stack_current(lltype.Signed) + # + # Load these variables from C code + start = _stack_get_start() + length = _stack_get_length() + # + # Common case: if 'current' is within [start:start+length], everything + # is fine + ofs = r_uint(current - start) + if ofs < r_uint(length): + return + # + # Else call the slow path + stack_check_slowpath(current) +stack_check._always_inline_ = True + + at rgc.no_collect +def stack_check_slowpath(current): + if ord(_stack_too_big_slowpath(current)): + # Now we are sure that the stack is really too big. Note that the # stack_unwind implementation is different depending on if stackless # is enabled. If it is it unwinds the stack, otherwise it simply # raises a RuntimeError. stack_unwind() +stack_check_slowpath._dont_inline_ = True # ____________________________________________________________ Modified: pypy/branch/fast-forward/pypy/rpython/lltypesystem/lloperation.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/lltypesystem/lloperation.py (original) +++ pypy/branch/fast-forward/pypy/rpython/lltypesystem/lloperation.py Tue Dec 14 00:05:06 2010 @@ -528,6 +528,8 @@ 'get_stack_depth_limit':LLOp(sideeffects=False), 'set_stack_depth_limit':LLOp(), + 'stack_current': LLOp(sideeffects=False), + # __________ misc operations __________ 'keepalive': LLOp(), Modified: pypy/branch/fast-forward/pypy/rpython/module/ll_os.py ============================================================================== --- pypy/branch/fast-forward/pypy/rpython/module/ll_os.py (original) +++ pypy/branch/fast-forward/pypy/rpython/module/ll_os.py Tue Dec 14 00:05:06 2010 @@ -99,7 +99,7 @@ return 'll_os.ll_os_w' + name def registering_str_unicode(posixfunc, condition=True): - if not condition: + if not condition or posixfunc is None: return registering(None, condition=False) func_name = posixfunc.__name__ @@ -1129,6 +1129,19 @@ return extdef([str, int, int], None, "ll_os.ll_os_chown", llimpl=os_chown_llimpl) + @registering_if(os, 'lchown') + def register_os_lchown(self): + os_lchown = self.llexternal('lchown',[rffi.CCHARP, rffi.INT, rffi.INT], + rffi.INT) + + def os_lchown_llimpl(path, uid, gid): + res = os_lchown(path, uid, gid) + if res == -1: + raise OSError(rposix.get_errno(), "os_lchown failed") + + return extdef([str, int, int], None, "ll_os.ll_os_lchown", + llimpl=os_lchown_llimpl) + @registering_if(os, 'readlink') def register_os_readlink(self): os_readlink = self.llexternal('readlink', @@ -1323,6 +1336,33 @@ return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, export_name=traits.ll_os_name('rename')) + @registering_str_unicode(getattr(os, 'mkfifo', None)) + def register_os_mkfifo(self, traits): + os_mkfifo = self.llexternal(traits.posix_function_name('mkfifo'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) + + def mkfifo_llimpl(path, mode): + res = rffi.cast(lltype.Signed, os_mkfifo(path, mode)) + if res < 0: + raise OSError(rposix.get_errno(), "os_mkfifo failed") + + return extdef([traits.str, int], s_None, llimpl=mkfifo_llimpl, + export_name=traits.ll_os_name('mkfifo')) + + @registering_str_unicode(getattr(os, 'mknod', None)) + def register_os_mknod(self, traits): + os_mknod = self.llexternal(traits.posix_function_name('mknod'), + [traits.CCHARP, rffi.MODE_T, rffi.INT], + rffi.INT) # xxx: actually ^^^ dev_t + + def mknod_llimpl(path, mode, dev): + res = rffi.cast(lltype.Signed, os_mknod(path, mode, dev)) + if res < 0: + raise OSError(rposix.get_errno(), "os_mknod failed") + + return extdef([traits.str, int, int], s_None, llimpl=mknod_llimpl, + export_name=traits.ll_os_name('mknod')) + @registering(os.umask) def register_os_umask(self): os_umask = self.llexternal(underscore_on_windows+'umask', [rffi.MODE_T], rffi.MODE_T) @@ -1348,6 +1388,20 @@ return extdef([int, int], s_None, llimpl=kill_llimpl, export_name="ll_os.ll_os_kill") + @registering_if(os, 'killpg') + def register_os_killpg(self): + os_killpg = self.llexternal('killpg', [rffi.INT, rffi.INT], + rffi.INT) + + def killpg_llimpl(pid, sig): + res = rffi.cast(lltype.Signed, os_killpg(rffi.cast(rffi.INT, pid), + rffi.cast(rffi.INT, sig))) + if res < 0: + raise OSError(rposix.get_errno(), "os_killpg failed") + + return extdef([int, int], s_None, llimpl=killpg_llimpl, + export_name="ll_os.ll_os_killpg") + @registering_if(os, 'link') def register_os_link(self): os_link = self.llexternal('link', [rffi.CCHARP, rffi.CCHARP], @@ -1444,6 +1498,25 @@ return extdef([int], s_None, llimpl=_exit_llimpl, export_name="ll_os.ll_os__exit") + @registering_if(os, 'nice') + def register_os_nice(self): + os_nice = self.llexternal('nice', [rffi.INT], rffi.INT) + + def nice_llimpl(inc): + # Assume that the system provides a standard-compliant version + # of nice() that returns the new priority. Nowadays, FreeBSD + # might be the last major non-compliant system (xxx check me). + rposix.set_errno(0) + res = rffi.cast(lltype.Signed, os_nice(inc)) + if res == -1: + err = rposix.get_errno() + if err != 0: + raise OSError(err, "os_nice failed") + return res + + return extdef([int], int, llimpl=nice_llimpl, + export_name="ll_os.ll_os_nice") + # --------------------------- os.stat & variants --------------------------- @registering(os.fstat) Modified: pypy/branch/fast-forward/pypy/translator/c/src/stack.h ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/src/stack.h (original) +++ pypy/branch/fast-forward/pypy/translator/c/src/stack.h Tue Dec 14 00:05:06 2010 @@ -11,27 +11,17 @@ * It is needed to have RPyThreadStaticTLS, too. */ #include "thread.h" +extern char *_LLstacktoobig_stack_start; + void LL_stack_unwind(void); -int LL_stack_too_big_slowpath(void); +char LL_stack_too_big_slowpath(long); /* returns 0 (ok) or 1 (too big) */ -extern volatile char *_LLstacktoobig_stack_base_pointer; -extern long _LLstacktoobig_stack_min; -extern long _LLstacktoobig_stack_max; +/* some macros referenced from pypy.rlib.rstack */ +#define OP_STACK_CURRENT(r) r = (long)&r +#define LL_stack_get_start() ((long)_LLstacktoobig_stack_start) +#define LL_stack_get_length() MAX_STACK_SIZE +#define LL_stack_get_start_adr() ((long)&_LLstacktoobig_stack_start) /* JIT */ -static int LL_stack_too_big(void) -{ - /* The fast path of stack_too_big, called extremely often. - Making it static makes an *inlinable* copy of this small - function's implementation in each compilation unit. */ - char local; - long diff = &local - _LLstacktoobig_stack_base_pointer; - /* common case: we are still in the same thread as last time - we checked, and still in the allowed part of the stack */ - return ((diff < _LLstacktoobig_stack_min || - diff > _LLstacktoobig_stack_max) - /* if not, call the slow path */ - && LL_stack_too_big_slowpath()); -} #ifdef __GNUC__ # define PYPY_INHIBIT_TAIL_CALL() asm("/* inhibit_tail_call */") @@ -61,68 +51,75 @@ return &local - parent; } -volatile char *_LLstacktoobig_stack_base_pointer = NULL; -long _LLstacktoobig_stack_min = 0; -long _LLstacktoobig_stack_max = 0; -RPyThreadStaticTLS _LLstacktoobig_stack_base_pointer_key; +char *_LLstacktoobig_stack_start = NULL; +int stack_direction = 0; +RPyThreadStaticTLS start_tls_key; -int LL_stack_too_big_slowpath(void) +char LL_stack_too_big_slowpath(long current) { - char local; long diff; - char *baseptr; - /* Check that the stack is less than MAX_STACK_SIZE bytes bigger - than the value recorded in stack_base_pointer. The base - pointer is updated to the current value if it is still NULL - or if we later find a &local that is below it. The real - stack base pointer is stored in thread-local storage, but we - try to minimize its overhead by keeping a local copy in - stack_pointer_pointer. */ + char *baseptr, *curptr = (char*)current; + + /* The stack_start variable is updated to match the current value + if it is still 0 or if we later find a 'curptr' position + that is below it. The real stack_start pointer is stored in + thread-local storage, but we try to minimize its overhead by + keeping a local copy in _LLstacktoobig_stack_start. */ - if (_LLstacktoobig_stack_min == _LLstacktoobig_stack_max /* == 0 */) { + if (stack_direction == 0) { /* not initialized */ /* XXX We assume that initialization is performed early, when there is still only one thread running. This allows us to ignore race conditions here */ - char *errmsg = RPyThreadStaticTLS_Create( - &_LLstacktoobig_stack_base_pointer_key); + char *errmsg = RPyThreadStaticTLS_Create(&start_tls_key); if (errmsg) { /* XXX should we exit the process? */ fprintf(stderr, "Internal PyPy error: %s\n", errmsg); return 1; } if (_LL_stack_growing_direction(NULL) > 0) - _LLstacktoobig_stack_max = MAX_STACK_SIZE; + stack_direction = +1; else - _LLstacktoobig_stack_min = -MAX_STACK_SIZE; + stack_direction = -1; } - baseptr = (char *) RPyThreadStaticTLS_Get( - _LLstacktoobig_stack_base_pointer_key); + baseptr = (char *) RPyThreadStaticTLS_Get(start_tls_key); if (baseptr != NULL) { - diff = &local - baseptr; - if (_LLstacktoobig_stack_min <= diff && - diff <= _LLstacktoobig_stack_max) { - /* within bounds */ - _LLstacktoobig_stack_base_pointer = baseptr; + diff = curptr - baseptr; + if (((unsigned long)diff) < (unsigned long)MAX_STACK_SIZE) { + /* within bounds, probably just had a thread switch */ + _LLstacktoobig_stack_start = baseptr; return 0; } - if ((_LLstacktoobig_stack_min == 0 && diff < 0) || - (_LLstacktoobig_stack_max == 0 && diff > 0)) { - /* we underflowed the stack, which means that - the initial estimation of the stack base must - be revised (see below) */ + if (stack_direction > 0) { + if (diff < 0 && diff > -MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } else { - return 1; /* stack overflow */ + if (diff >= MAX_STACK_SIZE && diff < 2*MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } + /* else we underflowed the stack, which means that + the initial estimation of the stack base must + be revised */ } /* update the stack base pointer to the current value */ - baseptr = &local; - RPyThreadStaticTLS_Set(_LLstacktoobig_stack_base_pointer_key, baseptr); - _LLstacktoobig_stack_base_pointer = baseptr; + if (stack_direction > 0) { + /* the valid range is [curptr:curptr+MAX_STACK_SIZE] */ + baseptr = curptr; + } + else { + /* the valid range is [curptr-MAX_STACK_SIZE+1:curptr+1] */ + baseptr = curptr - MAX_STACK_SIZE + 1; + } + RPyThreadStaticTLS_Set(start_tls_key, baseptr); + _LLstacktoobig_stack_start = baseptr; return 0; } Modified: pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py ============================================================================== --- pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py (original) +++ pypy/branch/fast-forward/pypy/translator/c/test/test_extfunc.py Tue Dec 14 00:05:06 2010 @@ -4,6 +4,7 @@ from pypy.tool.udir import udir from pypy.rlib.rarithmetic import r_longlong from pypy.translator.c.test.test_genc import compile +from pypy.translator.c.test.test_standalone import StandaloneTests posix = __import__(os.name) # note: clock synchronizes itself! @@ -404,6 +405,28 @@ assert os.path.exists(tmpfile2) assert not os.path.exists(tmpfile1) +if hasattr(os, 'mkfifo'): + def test_os_mkfifo(): + tmpfile = str(udir.join('test_os_mkfifo.txt')) + def does_stuff(): + os.mkfifo(tmpfile, 0666) + f1 = compile(does_stuff, []) + f1() + import stat + st = os.lstat(tmpfile) + assert stat.S_ISFIFO(st.st_mode) + +if hasattr(os, 'mknod'): + def test_os_mknod(): + import stat + tmpfile = str(udir.join('test_os_mknod.txt')) + def does_stuff(): + os.mknod(tmpfile, 0600 | stat.S_IFIFO, 0) + f1 = compile(does_stuff, []) + f1() + st = os.lstat(tmpfile) + assert stat.S_ISFIFO(st.st_mode) + def test_os_umask(): def does_stuff(): mask1 = os.umask(0660) @@ -516,6 +539,62 @@ # for what reason do they want us to shift by 8? See the doc assert status1 >> 8 == 4 +if hasattr(os, 'kill'): + def test_kill_to_send_sigusr1(): + import signal + from pypy.module.signal import interp_signal + def does_stuff(): + interp_signal.pypysig_setflag(signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR1) + interp_signal.pypysig_ignore(signal.SIGUSR1) + while True: + n = interp_signal.pypysig_poll() + if n < 0 or n == signal.SIGUSR1: + break + return n + f1 = compile(does_stuff, []) + got_signal = f1() + assert got_signal == signal.SIGUSR1 + +if hasattr(os, 'killpg'): + def test_killpg(): + import signal + from pypy.module.signal import interp_signal + def does_stuff(): + interp_signal.pypysig_setflag(signal.SIGUSR1) + os.killpg(os.getpgrp(), signal.SIGUSR1) + interp_signal.pypysig_ignore(signal.SIGUSR1) + while True: + n = interp_signal.pypysig_poll() + if n < 0 or n == signal.SIGUSR1: + break + return n + f1 = compile(does_stuff, []) + got_signal = f1() + assert got_signal == signal.SIGUSR1 + +if hasattr(os, 'chown') and hasattr(os, 'lchown'): + def test_os_chown_lchown(): + path1 = udir.join('test_os_chown_lchown-1.txt') + path2 = udir.join('test_os_chown_lchown-2.txt') + path1.write('foobar') + path2.mksymlinkto('some-broken-symlink') + tmpfile1 = str(path1) + tmpfile2 = str(path2) + def does_stuff(): + # xxx not really a test, just checks that they are callable + os.chown(tmpfile1, os.getuid(), os.getgid()) + os.lchown(tmpfile1, os.getuid(), os.getgid()) + os.lchown(tmpfile2, os.getuid(), os.getgid()) + try: + os.chown(tmpfile2, os.getuid(), os.getgid()) + except OSError: + pass + else: + raise AssertionError("os.chown(broken symlink) should raise") + f1 = compile(does_stuff, []) + f1() + # ____________________________________________________________ def _real_getenv(var): @@ -783,3 +862,19 @@ finally: os.chdir(localdir) assert res == True + +# ____________________________________________________________ + + +class TestExtFuncStandalone(StandaloneTests): + + if hasattr(os, 'nice'): + def test_os_nice(self): + def does_stuff(argv): + res = os.nice(3) + print 'os.nice returned', res + return 0 + t, cbuilder = self.compile(does_stuff) + data = cbuilder.cmdexec('') + res = os.nice(0) + 3 + assert data.startswith('os.nice returned %d\n' % res) From fijal at codespeak.net Tue Dec 14 07:30:22 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Dec 2010 07:30:22 +0100 (CET) Subject: [pypy-svn] r80047 - in pypy/branch/out-of-line-guards/pypy/jit: backend/llgraph metainterp metainterp/test Message-ID: <20101214063022.5E98350828@codespeak.net> Author: fijal Date: Tue Dec 14 07:30:19 2010 New Revision: 80047 Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/blackhole.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/memmgr.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_ztranslation.py pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmstate.py Log: Finish implementing out of line guards on llgraph backend. Has bugs though Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py Tue Dec 14 07:30:19 2010 @@ -164,6 +164,7 @@ class CompiledLoop(object): has_been_freed = False + invalidated = False def __init__(self): self.inputargs = [] @@ -294,6 +295,11 @@ assert not loop.has_been_freed loop.has_been_freed = True +def mark_as_invalid(loop): + loop = _from_opaque(loop) + assert not loop.has_been_freed + loop.invalidated = True + def compile_start_int_var(loop): return compile_start_ref_var(loop, lltype.Signed) @@ -927,7 +933,10 @@ raise GuardFailed def op_guard_not_invariant(self, descr): - pass + if self.loop.invalidated: + import pdb + pdb.set_trace() + raise GuardFailed class OOFrame(Frame): @@ -1641,6 +1650,7 @@ setannotation(compile_add_fail_arg, annmodel.s_None) setannotation(compile_redirect_fail, annmodel.s_None) setannotation(mark_as_free, annmodel.s_None) +setannotation(mark_as_invalid, annmodel.s_None) setannotation(new_frame, s_Frame) setannotation(frame_clear, annmodel.s_None) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py Tue Dec 14 07:30:19 2010 @@ -490,14 +490,18 @@ def get_invalidate_asm(self, TP, fieldname): def invalidate_asm(arg, fieldname): - prev = getattr(arg, fieldname) - next = prev + import pdb + pdb.set_trace() + next = getattr(arg, fieldname) while next: prev = next x = llmemory.weakref_deref(history.LoopToken._TYPE, prev.address) + if x: + x.invalidated = True + llimpl.mark_as_invalid( + x.compiled_loop_token.compiled_version) next = next.next - XXX # write me return invalidate_asm class OOtypeCPU_xxx_disabled(BaseCPU): Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/blackhole.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/blackhole.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/blackhole.py Tue Dec 14 07:30:19 2010 @@ -1087,9 +1087,15 @@ bhimpl_getfield_gc_r_pure = bhimpl_getfield_gc_r bhimpl_getfield_gc_f_pure = bhimpl_getfield_gc_f - bhimpl_getfield_gc_i_invariant = bhimpl_getfield_gc_i - bhimpl_getfield_gc_r_invariant = bhimpl_getfield_gc_r - bhimpl_getfield_gc_f_invariant = bhimpl_getfield_gc_f + @arguments("cpu", "r", "d", "i", returns="i") + def bhimpl_getfield_gc_i_invariant(cpu, struct, fielddescr, ignored): + return cpu.bh_getfield_gc_i(struct, fielddescr) + @arguments("cpu", "r", "d", "i", returns="r") + def bhimpl_getfield_gc_r_invariant(cpu, struct, fielddescr, ignored): + return cpu.bh_getfield_gc_r(struct, fielddescr) + @arguments("cpu", "r", "d", "i", returns="f") + def bhimpl_getfield_gc_f_invariant(cpu, struct, fielddescr, ignored): + return cpu.bh_getfield_gc_f(struct, fielddescr) bhimpl_getfield_vable_i = bhimpl_getfield_gc_i bhimpl_getfield_vable_r = bhimpl_getfield_gc_r @@ -1241,6 +1247,8 @@ # because of GUARD_NONNULL_CLASS. pass # + elif opnum == rop.GUARD_NOT_INVARIANT: + pass elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_NOT_FORCED): Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/compile.py Tue Dec 14 07:30:19 2010 @@ -564,7 +564,7 @@ # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr) prepare_last_operation(new_loop, target_loop_token) resumekey.compile_and_attach(metainterp, new_loop) - metainterp.remember_jit_invariants(target_loop_token) + metainterp.remember_jit_invariants(new_loop) record_loop_or_bridge(new_loop) return target_loop_token Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/history.py Tue Dec 14 07:30:19 2010 @@ -725,7 +725,7 @@ was compiled; but the LoopDescr remains alive and points to the generated assembler. """ - + # to make tests think we're dealing with an actual lltype _TYPE = lltype.Ptr(lltype.GcStruct('dummy struct for tests')) def _normalizedcontainer(self): @@ -738,6 +738,7 @@ _obj = property(_getobj) def _was_freed(self): return False + # terminating = False # see TerminatingLoopToken in compile.py outermost_jitdriver_sd = None @@ -751,6 +752,10 @@ # memory and the ResumeGuards. compiled_loop_token = None + invalidated = False + # if True means looptoken was invalidated by setting some field + # listed in jit_invariant_setfield that was read in this loop + def __init__(self): # For memory management of assembled loops self._keepalive_target_looktokens = {} # set of other LoopTokens Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/memmgr.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/memmgr.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/memmgr.py Tue Dec 14 07:30:19 2010 @@ -65,7 +65,8 @@ debug_print("Loop tokens before:", oldtotal) max_generation = self.current_generation - (self.max_age-1) for looptoken in self.alive_loops.keys(): - if 0 <= looptoken.generation < max_generation: + if (looptoken.invalidated or + 0 <= looptoken.generation < max_generation): del self.alive_loops[looptoken] newtotal = len(self.alive_loops) debug_print("Loop tokens freed: ", oldtotal - newtotal) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/pyjitpl.py Tue Dec 14 07:30:19 2010 @@ -1,5 +1,6 @@ import py, os, sys from pypy.rpython.lltypesystem import lltype, llmemory, rclass +from pypy.rpython.annlowlevel import cast_instance_to_base_ptr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -2238,7 +2239,11 @@ self.history.operations.append(op) def remember_jit_invariants(self, loop): - lltoken_weakref = llmemory.weakref_create(loop.token) + if we_are_translated(): + looptoken = cast_instance_to_base_ptr(loop.token) + else: + looptoken = loop.token + lltoken_weakref = llmemory.weakref_create(looptoken) seen = {} for b_struct, c_appender in self.invariant_structs: if (b_struct, c_appender) not in seen: Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_basic.py Tue Dec 14 07:30:19 2010 @@ -746,23 +746,27 @@ a = A() a.x = 1 - myjitdriver = JitDriver(greens = [], reds = ['i']) + myjitdriver = JitDriver(greens = [], reds = ['i', 'total']) @dont_look_inside def g(i): if i == 5: - a.x = 5 + a.x = 2 def f(): i = 0 - while i < 10: - myjitdriver.can_enter_jit(i=i) - myjitdriver.jit_merge_point(i=i) + total = 0 + while i < 20: + myjitdriver.can_enter_jit(i=i, total=total) + myjitdriver.jit_merge_point(i=i, total=total) g(i) i += a.x + total += i + return total - self.meta_interp(f, []) + assert self.meta_interp(f, []) == f() self.check_loop_count(2) + self.check_history(getfield_gc=0, getfield_gc_pure=0) def test_setfield_bool(self): class A: Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_ztranslation.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_ztranslation.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/test/test_ztranslation.py Tue Dec 14 07:30:19 2010 @@ -22,6 +22,7 @@ # - jitdriver hooks # - two JITs # - string concatenation, slicing and comparison + # - jit-invariants class Frame(object): _virtualizable2_ = ['i'] @@ -44,7 +45,20 @@ get_jitcell_at=get_jitcell_at, set_jitcell_at=set_jitcell_at, get_printable_location=get_printable_location) + + + class A(object): + _jit_invariant_fields_ = ['x'] + + @dont_look_inside + def g(i): + if i == 13: + prebuilt_a.x = 2 + + prebuilt_a = A() + def f(i): + prebuilt_a.x = 1 for param in unroll_parameters: defl = PARAMETERS[param] jitdriver.set_param(param, defl) @@ -58,7 +72,8 @@ total += frame.i if frame.i >= 20: frame.i -= 2 - frame.i -= 1 + g(frame.i) + frame.i -= prebuilt_a.x return total * 10 # myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x', 's']) Modified: pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmstate.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmstate.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/metainterp/warmstate.py Tue Dec 14 07:30:19 2010 @@ -158,7 +158,7 @@ if self.compiled_merge_points_wref is not None: for wref in self.compiled_merge_points_wref: looptoken = wref() - if looptoken is not None: + if looptoken is not None and not looptoken.invalidated: result.append(looptoken) return result @@ -460,6 +460,7 @@ cell = lltohlhack[rtyper.type_system.deref(cellref)] else: cell = None + # if not build: return cell if cell is None: From fijal at codespeak.net Tue Dec 14 07:39:06 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Dec 2010 07:39:06 +0100 (CET) Subject: [pypy-svn] r80048 - pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph Message-ID: <20101214063906.745A150828@codespeak.net> Author: fijal Date: Tue Dec 14 07:39:04 2010 New Revision: 80048 Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py Log: remove pdb (thanks alex) Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/llimpl.py Tue Dec 14 07:39:04 2010 @@ -934,8 +934,6 @@ def op_guard_not_invariant(self, descr): if self.loop.invalidated: - import pdb - pdb.set_trace() raise GuardFailed class OOFrame(Frame): Modified: pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py ============================================================================== --- pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py (original) +++ pypy/branch/out-of-line-guards/pypy/jit/backend/llgraph/runner.py Tue Dec 14 07:39:04 2010 @@ -490,8 +490,6 @@ def get_invalidate_asm(self, TP, fieldname): def invalidate_asm(arg, fieldname): - import pdb - pdb.set_trace() next = getattr(arg, fieldname) while next: prev = next From fijal at codespeak.net Tue Dec 14 07:52:13 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Dec 2010 07:52:13 +0100 (CET) Subject: [pypy-svn] r80049 - pypy/branch/remove-sys-recursionlimit/pypy/translator/benchmark Message-ID: <20101214065213.658D750828@codespeak.net> Author: fijal Date: Tue Dec 14 07:52:12 2010 New Revision: 80049 Removed: pypy/branch/remove-sys-recursionlimit/pypy/translator/benchmark/ Log: I think we don't use that any more for anything. Correct me if I'm wrong From fijal at codespeak.net Tue Dec 14 07:53:32 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Dec 2010 07:53:32 +0100 (CET) Subject: [pypy-svn] r80050 - pypy/branch/remove-sys-recursionlimit Message-ID: <20101214065332.1F45450828@codespeak.net> Author: fijal Date: Tue Dec 14 07:53:30 2010 New Revision: 80050 Removed: pypy/branch/remove-sys-recursionlimit/ Log: remove merged branch From fijal at codespeak.net Tue Dec 14 07:54:18 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Dec 2010 07:54:18 +0100 (CET) Subject: [pypy-svn] r80051 - pypy/trunk/pypy/translator/benchmark Message-ID: <20101214065418.7E8F450828@codespeak.net> Author: fijal Date: Tue Dec 14 07:54:17 2010 New Revision: 80051 Removed: pypy/trunk/pypy/translator/benchmark/ Log: I don't think we use that for anything any more. Delete (correct me if I'm wrong) From commits-noreply at bitbucket.org Tue Dec 14 09:17:13 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 02:17:13 -0600 (CST) Subject: [pypy-svn] buildbot commit cb83f6b653e6: handle nicely the tarballs generated by a mercurial-based buildbot Message-ID: <20101214081713.9BC85241231@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User Antonio Cuni # Date 1292314628 -3600 # Node ID cb83f6b653e624b671ea80fd59bca231db6e3a64 # Parent a7823765f22f7c3385b6392ab3744055d3eac61e handle nicely the tarballs generated by a mercurial-based buildbot --- a/bot2/pypybuildbot/pypylist.py +++ b/bot2/pypybuildbot/pypylist.py @@ -4,6 +4,7 @@ import itertools import re import py import cgi +import urllib from twisted.web import resource from twisted.web.static import File, DirectoryLister @@ -11,6 +12,11 @@ class PyPyTarball(object): # to get the desired order keep in mind that they are reversed at the end, # so the highest the value, the bigger the priority + VCS_PRIORITY = { + 'hg': 100, + 'svn': 50, + } + FEATURES_PRIORITY = { 'jit': 100, 'nojit': 50, @@ -42,10 +48,12 @@ class PyPyTarball(object): try: self.parse_filename() except ValueError: + self.vcs = None self.exe = None self.backend = None self.features = None self.rev = -1 + self.numrev = -1 self.platform = None def parse_filename(self): @@ -53,9 +61,18 @@ class PyPyTarball(object): raise ValueError name = self.filename.replace('.tar.bz2', '') self.exe, self.backend, self.features, self.rev, self.platform = name.split('-') + if ':' in self.rev: + # mercurial based + num, _ = self.rev.split(':') + self.numrev = int(num) + self.vcs = 'hg' + else: + self.numrev = int(self.rev) + self.vcs = 'svn' def key(self): - return (self.rev, + return (self.VCS_PRIORITY.get(self.vcs, -1), + self.numrev, self.FEATURES_PRIORITY.get(self.features, -1), self.PLATFORM_PRIORITY.get(self.platform, -1)) @@ -175,7 +192,7 @@ td,th {padding-left: 0.5em; padding-righ return tableContent def _add_test_results(self, element, rowClass): - filename = element['href'] + filename = urllib.unquote(element['href']) f = py.path.local(self.path).join(filename) date = datetime.date.fromtimestamp(f.mtime()) element['date'] = date.isoformat() --- a/bot2/pypybuildbot/test/test_pypylist.py +++ b/bot2/pypybuildbot/test/test_pypylist.py @@ -1,14 +1,27 @@ import py from pypybuildbot.pypylist import PyPyTarball -def test_pypytarball(): +def test_pypytarball_svn(): t = PyPyTarball('pypy-c-jit-75654-linux.tar.bz2') assert t.filename == 'pypy-c-jit-75654-linux.tar.bz2' assert t.exe == 'pypy' assert t.backend == 'c' assert t.features == 'jit' assert t.rev == '75654' + assert t.numrev == 75654 assert t.platform == 'linux' + assert t.vcs == 'svn' + +def test_pypytarball_hg(): + t = PyPyTarball('pypy-c-jit-75654:foo-linux.tar.bz2') + assert t.filename == 'pypy-c-jit-75654:foo-linux.tar.bz2' + assert t.exe == 'pypy' + assert t.backend == 'c' + assert t.features == 'jit' + assert t.rev == '75654:foo' + assert t.numrev == 75654 + assert t.platform == 'linux' + assert t.vcs == 'hg' def test_invalid_filename(): t = PyPyTarball('foo') @@ -29,11 +42,13 @@ def test_sort(): 'pypy-c-jit-10000-linux64.tar.bz2', 'pypy-c-jit-10000-win32.tar.bz2', 'pypy-c-stackless-10000-linux.tar.bz2', + 'pypy-c-jit-1000:e5b73981fc8d-linux.tar.bz2', # this is mercurial based ]) files.sort(key=PyPyTarball.key, reverse=True) files = [f.filename for f in files] assert files == [ + 'pypy-c-jit-1000:e5b73981fc8d-linux.tar.bz2', # mercurial first 'pypy-c-jit-20000-linux.tar.bz2', 'pypy-c-jit-10000-linux.tar.bz2', 'pypy-c-jit-10000-linux64.tar.bz2', From commits-noreply at bitbucket.org Tue Dec 14 09:24:15 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 02:24:15 -0600 (CST) Subject: [pypy-svn] buildbot commit e16954e1fdd3: fix test_pypylist Message-ID: <20101214082415.6D37F1E126B@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User Antonio Cuni # Date 1292315050 -3600 # Node ID e16954e1fdd3a37a263ae583fecac73e7dd8d374 # Parent cb83f6b653e624b671ea80fd59bca231db6e3a64 fix test_pypylist --- a/bot2/pypybuildbot/test/test_pypylist.py +++ b/bot2/pypybuildbot/test/test_pypylist.py @@ -25,6 +25,7 @@ def test_pypytarball_hg(): def test_invalid_filename(): t = PyPyTarball('foo') + assert t.vcs == None assert t.filename == 'foo' assert t.exe == None assert t.backend == None @@ -32,7 +33,7 @@ def test_invalid_filename(): assert t.rev == -1 assert t.platform == None t2 = PyPyTarball('pypy-c-jit-75654-linux.tar.bz2') - assert t < t2 + assert t.key() < t2.key() def test_sort(): files = map(PyPyTarball, [ @@ -81,7 +82,7 @@ def load_BuildmasterConfig(): def test_builder_names(): BuildmasterConfig = load_BuildmasterConfig() builders = [b['name'] for b in BuildmasterConfig['builders']] - known_exceptions = set(['own-win-x86-32']) + known_exceptions = set(['pypy-c-jit-macosx-x86-32']) def check_builder_names(t, expected_own, expected_app): own, app = t.get_builder_names() assert own == expected_own From commits-noreply at bitbucket.org Tue Dec 14 09:32:38 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 02:32:38 -0600 (CST) Subject: [pypy-svn] buildbot commit ebd81ebe66ca: fix all the tests that were broken by 13e8edc80748 Message-ID: <20101214083238.1291B6C1319@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User Antonio Cuni # Date 1292315546 -3600 # Node ID ebd81ebe66caa7a397b70e8daf588e90ca854f02 # Parent e16954e1fdd3a37a263ae583fecac73e7dd8d374 fix all the tests that were broken by 13e8edc80748 --- a/bot2/pypybuildbot/test/test_summary.py +++ b/bot2/pypybuildbot/test/test_summary.py @@ -693,28 +693,28 @@ class TestSummary(object): ['trunk', 'release/', 'branch/']) res = s._cat_branch_key(('foo', None)) - assert res == (0, 0, 0) + assert res == (0, 'foo', 0) res = s._cat_branch_key(('foo', 'trunk')) - assert res == (0, 0, 1, 'trunk') + assert res == (0, 'foo', 1, 'trunk') res = s._cat_branch_key(('bar', 'trunk')) - assert res == (0, 1, 1, 'trunk') + assert res == (1, 'bar', 1, 'trunk') - res = s._cat_branch_key((None, 'trunk')) - assert res == (1, None, 1, 'trunk') + res = s._cat_branch_key(('', 'trunk')) + assert res == (2, '', 1, 'trunk') res = s._cat_branch_key(('dontknow', 'trunk')) - assert res == (1, 'dontknow', 1, 'trunk') + assert res == (2, 'dontknow', 1, 'trunk') - res = s._cat_branch_key((None, 'branch/foo')) - assert res == (1, None, 3, 'branch/foo') + res = s._cat_branch_key(('', 'branch/foo')) + assert res == (2, '', 3, 'branch/foo') - res = s._cat_branch_key((None, 'release/1')) - assert res == (1, None, 2, 'release/1') + res = s._cat_branch_key(('', 'release/1')) + assert res == (2, '', 2, 'release/1') - res = s._cat_branch_key((None, 'what')) - assert res == (1, None, 4, 'what') + res = s._cat_branch_key(('', 'what')) + assert res == (2, '', 4, 'what') def test_builders_with_categories(self): builder1 = status_builder.BuilderStatus('builder_foo') @@ -722,7 +722,7 @@ class TestSummary(object): builder2 = status_builder.BuilderStatus('builder_bar') builder2.category = 'bar' builder3 = status_builder.BuilderStatus('builder_') - builder3.category = None + builder3.category = '' add_builds(builder1, [('60000', "F TEST1\n")]) add_builds(builder2, [('60000', "F TEST2\n")]) From agaynor at codespeak.net Tue Dec 14 10:05:15 2010 From: agaynor at codespeak.net (agaynor at codespeak.net) Date: Tue, 14 Dec 2010 10:05:15 +0100 (CET) Subject: [pypy-svn] r80052 - pypy/trunk/pypy/interpreter Message-ID: <20101214090515.2F1F850834@codespeak.net> Author: agaynor Date: Tue Dec 14 10:05:13 2010 New Revision: 80052 Modified: pypy/trunk/pypy/interpreter/baseobjspace.py Log: Make c_filedescriptor_w more closely mirror CPython's PyObject_AsFileDescriptor (in behavior and structure), which doesn't bother verifying the int fits in 32-bits. Modified: pypy/trunk/pypy/interpreter/baseobjspace.py ============================================================================== --- pypy/trunk/pypy/interpreter/baseobjspace.py (original) +++ pypy/trunk/pypy/interpreter/baseobjspace.py Tue Dec 14 10:05:13 2010 @@ -797,8 +797,8 @@ def call_obj_args(self, w_callable, w_obj, args): if not self.config.objspace.disable_call_speedhacks: - # XXX start of hack for performance - from pypy.interpreter.function import Function + # XXX start of hack for performance + from pypy.interpreter.function import Function if isinstance(w_callable, Function): return w_callable.call_obj_args(w_obj, args) # XXX end of hack for performance @@ -1184,24 +1184,27 @@ return value def c_filedescriptor_w(self, w_fd): - try: - fd = self.c_int_w(w_fd) - except OperationError, e: - if not e.match(self, self.w_TypeError): - raise + if (not self.isinstance_w(w_fd, self.w_int) and + not self.isinstance_w(w_fd, self.w_long)): try: - w_fileno = self.getattr(w_fd, self.wrap('fileno')) + w_fileno = self.getattr(w_fd, self.wrap("fileno")) except OperationError, e: if e.match(self, self.w_AttributeError): raise OperationError(self.w_TypeError, - self.wrap("argument must be an int, " - "or have a fileno() method.")) + self.wrap("argument must be an int, or have a fileno() " + "method.") + ) raise w_fd = self.call_function(w_fileno) - fd = self.c_int_w(w_fd) + if not self.isinstance_w(w_fd, self.w_int): + raise OperationError(self.w_TypeError, + self.wrap("fileno() must return an integer") + ) + fd = self.int_w(w_fd) if fd < 0: raise operationerrfmt(self.w_ValueError, - "file descriptor cannot be a negative integer (%d)", fd) + "file descriptor cannot be a negative integer (%d)", fd + ) return fd def warn(self, msg, w_warningcls): @@ -1403,4 +1406,3 @@ 'call_args', 'marshal_w', ] - From commits-noreply at bitbucket.org Tue Dec 14 10:23:56 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 03:23:56 -0600 (CST) Subject: [pypy-svn] pypy commit a1d660058003: I don't 100% understand what's going on. It seems that if you have a Message-ID: <20101214092356.B3FA46C1070@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292318594 -3600 # Node ID a1d660058003b0c82e864c81be80b8093855cfc0 # Parent e5b73981fc8d5cd14074426b0e439f0716379a31 I don't 100% understand what's going on. It seems that if you have a conftest.py in the test/ directory, py.test won't call the pytest_collect_directory hook. However, if will call it if it is in the parent directory. Before, everything worked because we had a .svn dir everywhere, so the conftest was in the parent dir of the .svn (!!!). Fix it by moving the conftests one level up. --- /dev/null +++ b/pypy/module/test_lib_pypy/conftest.py @@ -0,0 +1,25 @@ +import py +import sys + +def pytest_collect_directory(): + if '__pypy__' not in sys.builtin_module_names: + py.test.skip("these tests are meant to be run on top of pypy-c") + +def compile_so_file(): + from pypy.translator.platform import platform + from pypy.translator.tool.cbuild import ExternalCompilationInfo + udir = py.test.ensuretemp('_ctypes_test') + cfile = py.path.local(__file__).dirpath().join("ctypes_tests", "_ctypes_test.c") + + if sys.platform == 'win32': + libraries = ['oleaut32'] + else: + libraries = [] + eci = ExternalCompilationInfo(libraries=libraries) + + return platform.compile([cfile], eci, str(udir.join('_ctypes_test')), + standalone=False) + +def pytest_configure(config): + global sofile + sofile = compile_so_file() --- /dev/null +++ b/pypy/rlib/rsdl/conftest.py @@ -0,0 +1,8 @@ +from pypy.rlib.rsdl.eci import check_sdl_installation, SDLNotInstalled +import py + +def pytest_collect_directory(): + try: + check_sdl_installation() + except SDLNotInstalled, e: + py.test.skip("SDL not installed(?): %s" % (e,)) --- a/pypy/module/test_lib_pypy/ctypes_tests/conftest.py +++ /dev/null @@ -1,25 +0,0 @@ -import py -import sys - -def pytest_collect_directory(): - if '__pypy__' not in sys.builtin_module_names: - py.test.skip("these tests are meant to be run on top of pypy-c") - -def compile_so_file(): - from pypy.translator.platform import platform - from pypy.translator.tool.cbuild import ExternalCompilationInfo - udir = py.test.ensuretemp('_ctypes_test') - cfile = py.path.local(__file__).dirpath().join("_ctypes_test.c") - - if sys.platform == 'win32': - libraries = ['oleaut32'] - else: - libraries = [] - eci = ExternalCompilationInfo(libraries=libraries) - - return platform.compile([cfile], eci, str(udir.join('_ctypes_test')), - standalone=False) - -def pytest_configure(config): - global sofile - sofile = compile_so_file() --- a/pypy/jit/backend/llvm/test/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -import py - -def pytest_collect_directory(): - py.test.skip("llvm backend tests skipped for now") --- a/pypy/rlib/rsdl/test/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -from pypy.rlib.rsdl.eci import check_sdl_installation, SDLNotInstalled -import py - -def pytest_collect_directory(): - try: - check_sdl_installation() - except SDLNotInstalled, e: - py.test.skip("SDL not installed(?): %s" % (e,)) --- /dev/null +++ b/pypy/jit/backend/llvm/conftest.py @@ -0,0 +1,4 @@ +import py + +def pytest_collect_directory(): + py.test.skip("llvm backend tests skipped for now") From commits-noreply at bitbucket.org Tue Dec 14 11:24:29 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:24:29 -0600 (CST) Subject: [pypy-svn] pypy commit f05147cddb71: Delete bogus lines ending in a slash from '.hgignore'. Message-ID: <20101214102429.CDC44241085@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Dan Villiom Podlaski Christiansen # Date 1292322242 -3600 # Node ID f05147cddb7176830601931fc023a556842a35ca # Parent a1d660058003b0c82e864c81be80b8093855cfc0 Delete bogus lines ending in a slash from '.hgignore'. The cause of these lines is a bug in hgsubversion due to incorrect handling of empty lines in the svn:ignore property. --- a/.hgignore +++ b/.hgignore @@ -6,7 +6,6 @@ site-packages pypy/module/cpyext/src/*.o pypy/bin/pypy-c pypy/translator/jvm/src/pypy/*.class -pypy/module/_stackless/test/ pypy/module/cpyext/test/*.errors pypy/doc/*.html pypy/doc/basicblock.asc @@ -20,7 +19,6 @@ pypy/translator/benchmark/gadfly pypy/translator/benchmark/mako pypy/translator/benchmark/bench-custom.benchmark_result pypy/translator/benchmark/shootout_benchmarks -pypy/module/_stackless/ pypy/translator/goal/pypy-translation-snapshot pypy/translator/goal/pypy-c* pypy/translator/goal/*.exe @@ -38,12 +36,10 @@ lib_pypy/ctypes_config_cache/_*_cache.py lib_pypy/ctypes_config_cache/_*_*_.py pypy/translator/cli/query-descriptions pypy/doc/discussion/*.html -pypy/doc/discussion/ include/*.h include/*.inl pypy/doc/config/*.html pypy/doc/config/style.css -pypy/doc/config/ pypy/doc/jit/*.html pypy/doc/jit/style.css pypy/doc/image/lattice1.png From commits-noreply at bitbucket.org Tue Dec 14 11:41:49 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:49 -0600 (CST) Subject: [pypy-svn] pypy commit 9c2aca640d70: Next register allocation fix Message-ID: <20101214104149.C1DAB241316@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User David Schneider # Date 1292266968 0 # Node ID 9c2aca640d703d6152e4223d7e6746c4391ceb59 # Parent c1dae116dd3ea2d26b66e5324f99029203113ec2 Next register allocation fix --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -143,8 +143,8 @@ class ARMRegisterManager(RegisterManager boxes.append(box) l1, box = self._ensure_value_is_boxed(a1, [box]) boxes.append(box) - res = self.force_allocate_reg(op.result, boxes) self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) self.possibly_free_var(op.result) return [l0, l1, res] @@ -167,8 +167,8 @@ class ARMRegisterManager(RegisterManager boxes.append(box) l1, box = self._ensure_value_is_boxed(a1, boxes) boxes.append(box) - res = self.force_allocate_reg(op.result, boxes) self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) self.possibly_free_var(op.result) return [l0, l1, res] @@ -181,8 +181,8 @@ class ARMRegisterManager(RegisterManager reg2, box = self._ensure_value_is_boxed(a1, forbidden_vars=boxes) boxes.append(box) - res = self.force_allocate_reg(op.result, boxes) self.possibly_free_vars(boxes) + res = self.force_allocate_reg(op.result) self.possibly_free_var(op.result) return [reg1, reg2, res] @@ -241,8 +241,9 @@ class ARMRegisterManager(RegisterManager def prepare_op_int_neg(self, op, fcond): l0, box = self._ensure_value_is_boxed(op.getarg(0)) - resloc = self.force_allocate_reg(op.result, [box]) - self.possibly_free_vars([box, op.result]) + self.possibly_free_var(box) + resloc = self.force_allocate_reg(op.result) + self.possibly_free_var(op.result) return [l0, resloc] prepare_op_int_invert = prepare_op_int_neg @@ -376,7 +377,7 @@ class ARMRegisterManager(RegisterManager base_loc, base_box = self._ensure_value_is_boxed(a0) self.possibly_free_var(a0) self.possibly_free_var(base_box) - res = self.force_allocate_reg(op.result, [a0]) + res = self.force_allocate_reg(op.result) self.possibly_free_var(op.result) return [base_loc, imm(ofs), res, imm(size)] --- a/pypy/jit/backend/arm/helper/regalloc.py +++ b/pypy/jit/backend/arm/helper/regalloc.py @@ -49,13 +49,10 @@ def prepare_op_by_helper_call(): arg2 = self.make_sure_var_in_reg(a1, selected_reg=r.r1) assert arg1 == r.r0 assert arg2 == r.r1 - spilled = False if isinstance(a0, Box) and self.stays_alive(a0): - spilled = True self.force_spill_var(a0) + self.possibly_free_var(a0) self.after_call(op.result) - if spilled: - self.possibly_free_var(a0) self.possibly_free_var(a1) self.possibly_free_var(op.result) return [] From commits-noreply at bitbucket.org Tue Dec 14 11:41:51 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:51 -0600 (CST) Subject: [pypy-svn] pypy commit c9e8ed0461c7: fixed tests Message-ID: <20101214104151.2DA71241316@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Hakan Ardo # Date 1292270030 0 # Node ID c9e8ed0461c7150566557861a21f2984ab6dc78d # Parent f8a367dd165594fb308556aff40b66869cec2bb8 fixed tests --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -113,12 +113,12 @@ def compile_new_loop(metainterp, old_loo if loop.preamble.operations is not None: send_loop_to_backend(metainterp_sd, loop, "loop") record_loop_or_bridge(loop) + token = loop.preamble.token if full_preamble_needed or not loop.preamble.token.short_preamble: send_loop_to_backend(metainterp_sd, loop.preamble, "entry bridge") insert_loop_token(old_loop_tokens, loop.preamble.token) jitdriver_sd.warmstate.attach_unoptimized_bridge_from_interp( greenkey, loop.preamble.token) - token = loop.preamble.token record_loop_or_bridge(loop.preamble) return token else: --- a/pypy/jit/metainterp/test/test_memmgr.py +++ b/pypy/jit/metainterp/test/test_memmgr.py @@ -132,7 +132,7 @@ class _TestIntegration(LLJitMixin): res = self.meta_interp(f, [], loop_longevity=1) assert res == 42 # we should see a loop for each call to g() - self.check_tree_loop_count(8 + 20*2) + self.check_tree_loop_count(8 + 20*2*2) def test_throw_away_old_loops(self): myjitdriver = JitDriver(greens=['m'], reds=['n']) @@ -157,7 +157,7 @@ class _TestIntegration(LLJitMixin): res = self.meta_interp(f, [], loop_longevity=3) assert res == 42 - self.check_tree_loop_count(2 + 10*4) # 42 :-) + self.check_tree_loop_count(2 + 10*4*2) def test_call_assembler_keep_alive(self): myjitdriver1 = JitDriver(greens=['m'], reds=['n']) @@ -191,7 +191,7 @@ class _TestIntegration(LLJitMixin): res = self.meta_interp(f, [1], loop_longevity=4, inline=True) assert res == 42 - self.check_tree_loop_count(8) + self.check_tree_loop_count(12) # ____________________________________________________________ --- a/pypy/module/pypyjit/test/test_pypy_c.py +++ b/pypy/module/pypyjit/test/test_pypy_c.py @@ -422,7 +422,7 @@ class PyPyCJITTests(object): ([1000], 49500), ([10000], 495000), ([100000], 4950000)) - assert len(self.loops) == 3 + assert len(self.rawloops) + len(self.rawentrybridges) == 4 op, = self.get_by_bytecode("CALL_FUNCTION_KW") # XXX a bit too many guards, but better than before assert len(op.get_opnames("guard")) <= 12 --- a/pypy/jit/metainterp/test/test_optimizebasic.py +++ b/pypy/jit/metainterp/test/test_optimizebasic.py @@ -262,12 +262,14 @@ class BaseTestBasic(BaseTest): OptString, OptHeap, Optimizer) + from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall + optimizations = [OptIntBounds(), OptRewrite(), OptVirtualize(), OptString(), OptHeap(), - #OptFfiCall(), + OptFfiCall(), ] optimizer = Optimizer(metainterp_sd, loop, optimizations) optimizer.propagate_all_forward() From commits-noreply at bitbucket.org Tue Dec 14 11:41:52 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:52 -0600 (CST) Subject: [pypy-svn] pypy commit 69ac5b5dc4ac: Finish implementing out of line guards on llgraph backend. Has bugs though Message-ID: <20101214104152.40C76241316@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Maciej Fijalkowski # Date 1292308219 0 # Node ID 69ac5b5dc4acc42177b63c69dbbcc25989e7e537 # Parent e407bdf735c5d1d48f6aef319f0822b0dc8a1e0a Finish implementing out of line guards on llgraph backend. Has bugs though --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -564,7 +564,7 @@ def compile_new_bridge(metainterp, old_l # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr) prepare_last_operation(new_loop, target_loop_token) resumekey.compile_and_attach(metainterp, new_loop) - metainterp.remember_jit_invariants(target_loop_token) + metainterp.remember_jit_invariants(new_loop) record_loop_or_bridge(new_loop) return target_loop_token --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -490,14 +490,18 @@ class LLtypeCPU(BaseCPU): def get_invalidate_asm(self, TP, fieldname): def invalidate_asm(arg, fieldname): - prev = getattr(arg, fieldname) - next = prev + import pdb + pdb.set_trace() + next = getattr(arg, fieldname) while next: prev = next x = llmemory.weakref_deref(history.LoopToken._TYPE, prev.address) + if x: + x.invalidated = True + llimpl.mark_as_invalid( + x.compiled_loop_token.compiled_version) next = next.next - XXX # write me return invalidate_asm class OOtypeCPU_xxx_disabled(BaseCPU): --- a/pypy/jit/metainterp/history.py +++ b/pypy/jit/metainterp/history.py @@ -725,7 +725,7 @@ class LoopToken(AbstractDescr): was compiled; but the LoopDescr remains alive and points to the generated assembler. """ - + # to make tests think we're dealing with an actual lltype _TYPE = lltype.Ptr(lltype.GcStruct('dummy struct for tests')) def _normalizedcontainer(self): @@ -738,6 +738,7 @@ class LoopToken(AbstractDescr): _obj = property(_getobj) def _was_freed(self): return False + # terminating = False # see TerminatingLoopToken in compile.py outermost_jitdriver_sd = None @@ -751,6 +752,10 @@ class LoopToken(AbstractDescr): # memory and the ResumeGuards. compiled_loop_token = None + invalidated = False + # if True means looptoken was invalidated by setting some field + # listed in jit_invariant_setfield that was read in this loop + def __init__(self): # For memory management of assembled loops self._keepalive_target_looktokens = {} # set of other LoopTokens --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1,5 +1,6 @@ import py, os, sys from pypy.rpython.lltypesystem import lltype, llmemory, rclass +from pypy.rpython.annlowlevel import cast_instance_to_base_ptr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.debug import debug_start, debug_stop, debug_print @@ -2238,7 +2239,11 @@ class MetaInterp(object): self.history.operations.append(op) def remember_jit_invariants(self, loop): - lltoken_weakref = llmemory.weakref_create(loop.token) + if we_are_translated(): + looptoken = cast_instance_to_base_ptr(loop.token) + else: + looptoken = loop.token + lltoken_weakref = llmemory.weakref_create(looptoken) seen = {} for b_struct, c_appender in self.invariant_structs: if (b_struct, c_appender) not in seen: --- a/pypy/jit/metainterp/warmstate.py +++ b/pypy/jit/metainterp/warmstate.py @@ -158,7 +158,7 @@ class JitCell(BaseJitCell): if self.compiled_merge_points_wref is not None: for wref in self.compiled_merge_points_wref: looptoken = wref() - if looptoken is not None: + if looptoken is not None and not looptoken.invalidated: result.append(looptoken) return result @@ -460,6 +460,7 @@ class WarmEnterState(object): cell = lltohlhack[rtyper.type_system.deref(cellref)] else: cell = None + # if not build: return cell if cell is None: --- a/pypy/jit/metainterp/blackhole.py +++ b/pypy/jit/metainterp/blackhole.py @@ -1087,9 +1087,15 @@ class BlackholeInterpreter(object): bhimpl_getfield_gc_r_pure = bhimpl_getfield_gc_r bhimpl_getfield_gc_f_pure = bhimpl_getfield_gc_f - bhimpl_getfield_gc_i_invariant = bhimpl_getfield_gc_i - bhimpl_getfield_gc_r_invariant = bhimpl_getfield_gc_r - bhimpl_getfield_gc_f_invariant = bhimpl_getfield_gc_f + @arguments("cpu", "r", "d", "i", returns="i") + def bhimpl_getfield_gc_i_invariant(cpu, struct, fielddescr, ignored): + return cpu.bh_getfield_gc_i(struct, fielddescr) + @arguments("cpu", "r", "d", "i", returns="r") + def bhimpl_getfield_gc_r_invariant(cpu, struct, fielddescr, ignored): + return cpu.bh_getfield_gc_r(struct, fielddescr) + @arguments("cpu", "r", "d", "i", returns="f") + def bhimpl_getfield_gc_f_invariant(cpu, struct, fielddescr, ignored): + return cpu.bh_getfield_gc_f(struct, fielddescr) bhimpl_getfield_vable_i = bhimpl_getfield_gc_i bhimpl_getfield_vable_r = bhimpl_getfield_gc_r @@ -1241,6 +1247,8 @@ class BlackholeInterpreter(object): # because of GUARD_NONNULL_CLASS. pass # + elif opnum == rop.GUARD_NOT_INVARIANT: + pass elif (opnum == rop.GUARD_NO_EXCEPTION or opnum == rop.GUARD_EXCEPTION or opnum == rop.GUARD_NOT_FORCED): --- a/pypy/jit/metainterp/memmgr.py +++ b/pypy/jit/metainterp/memmgr.py @@ -65,7 +65,8 @@ class MemoryManager(object): debug_print("Loop tokens before:", oldtotal) max_generation = self.current_generation - (self.max_age-1) for looptoken in self.alive_loops.keys(): - if 0 <= looptoken.generation < max_generation: + if (looptoken.invalidated or + 0 <= looptoken.generation < max_generation): del self.alive_loops[looptoken] newtotal = len(self.alive_loops) debug_print("Loop tokens freed: ", oldtotal - newtotal) --- a/pypy/jit/backend/llgraph/llimpl.py +++ b/pypy/jit/backend/llgraph/llimpl.py @@ -164,6 +164,7 @@ TYPES = { class CompiledLoop(object): has_been_freed = False + invalidated = False def __init__(self): self.inputargs = [] @@ -294,6 +295,11 @@ def mark_as_free(loop): assert not loop.has_been_freed loop.has_been_freed = True +def mark_as_invalid(loop): + loop = _from_opaque(loop) + assert not loop.has_been_freed + loop.invalidated = True + def compile_start_int_var(loop): return compile_start_ref_var(loop, lltype.Signed) @@ -927,7 +933,10 @@ class Frame(object): raise GuardFailed def op_guard_not_invariant(self, descr): - pass + if self.loop.invalidated: + import pdb + pdb.set_trace() + raise GuardFailed class OOFrame(Frame): @@ -1641,6 +1650,7 @@ setannotation(compile_add_fail, annmodel setannotation(compile_add_fail_arg, annmodel.s_None) setannotation(compile_redirect_fail, annmodel.s_None) setannotation(mark_as_free, annmodel.s_None) +setannotation(mark_as_invalid, annmodel.s_None) setannotation(new_frame, s_Frame) setannotation(frame_clear, annmodel.s_None) --- a/pypy/jit/metainterp/test/test_basic.py +++ b/pypy/jit/metainterp/test/test_basic.py @@ -746,23 +746,27 @@ class BasicTests: a = A() a.x = 1 - myjitdriver = JitDriver(greens = [], reds = ['i']) + myjitdriver = JitDriver(greens = [], reds = ['i', 'total']) @dont_look_inside def g(i): if i == 5: - a.x = 5 + a.x = 2 def f(): i = 0 - while i < 10: - myjitdriver.can_enter_jit(i=i) - myjitdriver.jit_merge_point(i=i) + total = 0 + while i < 20: + myjitdriver.can_enter_jit(i=i, total=total) + myjitdriver.jit_merge_point(i=i, total=total) g(i) i += a.x + total += i + return total - self.meta_interp(f, []) + assert self.meta_interp(f, []) == f() self.check_loop_count(2) + self.check_history(getfield_gc=0, getfield_gc_pure=0) def test_setfield_bool(self): class A: --- a/pypy/jit/metainterp/test/test_ztranslation.py +++ b/pypy/jit/metainterp/test/test_ztranslation.py @@ -22,6 +22,7 @@ class TranslationTest: # - jitdriver hooks # - two JITs # - string concatenation, slicing and comparison + # - jit-invariants class Frame(object): _virtualizable2_ = ['i'] @@ -44,7 +45,20 @@ class TranslationTest: get_jitcell_at=get_jitcell_at, set_jitcell_at=set_jitcell_at, get_printable_location=get_printable_location) + + + class A(object): + _jit_invariant_fields_ = ['x'] + + @dont_look_inside + def g(i): + if i == 13: + prebuilt_a.x = 2 + + prebuilt_a = A() + def f(i): + prebuilt_a.x = 1 for param in unroll_parameters: defl = PARAMETERS[param] jitdriver.set_param(param, defl) @@ -58,7 +72,8 @@ class TranslationTest: total += frame.i if frame.i >= 20: frame.i -= 2 - frame.i -= 1 + g(frame.i) + frame.i -= prebuilt_a.x return total * 10 # myjitdriver2 = JitDriver(greens = ['g'], reds = ['m', 'x', 's']) From commits-noreply at bitbucket.org Tue Dec 14 11:41:52 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:52 -0600 (CST) Subject: [pypy-svn] pypy commit 55d92d35c5b4: remove pdb (thanks alex) Message-ID: <20101214104152.F11DA241416@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Maciej Fijalkowski # Date 1292308744 0 # Node ID 55d92d35c5b4a4a329fd165cd05f902db29aa894 # Parent 69ac5b5dc4acc42177b63c69dbbcc25989e7e537 remove pdb (thanks alex) --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -490,8 +490,6 @@ class LLtypeCPU(BaseCPU): def get_invalidate_asm(self, TP, fieldname): def invalidate_asm(arg, fieldname): - import pdb - pdb.set_trace() next = getattr(arg, fieldname) while next: prev = next --- a/pypy/jit/backend/llgraph/llimpl.py +++ b/pypy/jit/backend/llgraph/llimpl.py @@ -934,8 +934,6 @@ class Frame(object): def op_guard_not_invariant(self, descr): if self.loop.invalidated: - import pdb - pdb.set_trace() raise GuardFailed class OOFrame(Frame): From commits-noreply at bitbucket.org Tue Dec 14 11:41:53 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:53 -0600 (CST) Subject: [pypy-svn] pypy commit 9a0936e25c6f: I think we don't use that any more for anything. Correct me if I'm wrong Message-ID: <20101214104153.97FB0241316@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Maciej Fijalkowski # Date 1292309532 0 # Node ID 9a0936e25c6f4aa5a4d0cfe30455ea29907e4d5a # Parent 8d3fa242195306534a48aa3fdf49d26f413915ab I think we don't use that any more for anything. Correct me if I'm wrong --- a/pypy/translator/benchmark/autopath.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') -import py # note: py is imported only AFTER the path has been set -libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) -libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) - -if __name__ == '__main__': - __clone() --- a/pypy/translator/benchmark/bench-custom.py +++ /dev/null @@ -1,127 +0,0 @@ -# benchmarks on a unix machine. - -import autopath -from pypy.translator.benchmark.result import BenchmarkResultSet -from pypy.translator.benchmark.benchmarks import BENCHMARKS -import os, sys, time, pickle, re, py - -SPLIT_TABLE = True # useful when executable names are very long - -def get_executables(args): #sorted by revision number (highest first) - exes = sorted(args, key=os.path.getmtime) - r = [] - for exe in exes: - if '/' not in exe: - r.append('./' + exe) - else: - r.append(exe) - return r - -def main(options, args): - if os.path.exists(options.picklefile): - benchmark_result = pickle.load(open(options.picklefile, 'rb')) - else: - benchmark_result = BenchmarkResultSet() - - benchmarks = [] - for b in BENCHMARKS: - if b.name in options.benchmarks: - if not b.check(): - print "can't run %s benchmark for some reason"%(b.name,) - else: - if int(options.sizefactor) > 1: - b = b * int(options.sizefactor) - benchmarks.append(b) - - exes = get_executables(args) - pythons = 'python2.6 python2.5 python2.4'.split() - full_pythons = [] - for python in pythons: - full_python = py.path.local.sysfind(python) - if full_python: - full_pythons.append(str(full_python)) - - sys.stdout.flush() - - refs = {} - final_error_count = 0 - - if not options.nocpython: - exes = full_pythons + exes - - for i in range(int(options.runcount)) or [None]: - if i is not None: - for exe in exes: - for b in benchmarks: - br = benchmark_result.result(exe, allowcreate=True) - result = br.run_benchmark(b, verbose=options.verbose) - if not result: - final_error_count += 1 - - if options.relto: - relto = options.relto - else: - relto = full_pythons[0] - if relto not in benchmark_result.benchmarks: - continue - - pickle.dump(benchmark_result, open(options.picklefile, 'wb')) - - exe_stats = ['stat:st_mtime', 'exe_name', 'pypy_rev'] - if not SPLIT_TABLE: - stats = exe_stats[:] - else: - stats = ['exe'] - for b in benchmarks: - stats.append('bench:'+b.name) - kwds = {'relto': relto, - 'filteron' :lambda r: r.exe_name in exes, - } - for row in benchmark_result.txt_summary(stats, **kwds): - print row - if SPLIT_TABLE: - print - print 'Reference:' - for row in benchmark_result.txt_summary(['exe'] + exe_stats, - **kwds): - print row - print - - if final_error_count: - raise SystemExit("%d benchmark run(s) failed (see -FAILED- above)" - % final_error_count) - -if __name__ == '__main__': - from optparse import OptionParser - parser = OptionParser() - default_benches = ','.join([b.name for b in BENCHMARKS if b.check()]) - parser.add_option( - '--benchmarks', dest='benchmarks', - default=default_benches, - ) - parser.add_option( - '--pickle', dest='picklefile', - default='bench-custom.benchmark_result' - ) - parser.add_option( - '--runcount', dest='runcount', - default='1', - ) - parser.add_option( - '--relto', dest='relto', - default=None, - ) - parser.add_option( - '-v', '--verbose', action='store_true', dest='verbose', - default=None, - ) - parser.add_option( - '--no-cpython', action='store_true', dest='nocpython', - default=None, - ) - parser.add_option( - '--size-factor', dest='sizefactor', - default='1', - ) - options, args = parser.parse_args(sys.argv[1:]) - main(options, args) --- a/pypy/translator/benchmark/__init__.py +++ /dev/null @@ -1,1 +0,0 @@ -# --- a/pypy/translator/benchmark/benchmarks.py +++ /dev/null @@ -1,197 +0,0 @@ -import os, sys, time, pickle, re, py -import yaml - -class BenchmarkFailed(Exception): - pass - -PYSTONE_CMD = 'from test import pystone;pystone.main(%s)' -PYSTONE_PATTERN = 'This machine benchmarks at' - -RICHARDS_CMD = 'from richards import *;main(iterations=%d)' -RICHARDS_PATTERN = 'Average time per iteration:' - -TIME_FMT = 'max mem used: %Mk\nelapsed time: %e\nsystem time: %S\nuser time: %U\nCPU use: %P' - -def get_result(txt, pattern): - for line in txt.split('\n'): - if line.startswith(pattern): - break - else: - raise BenchmarkFailed - return float(line.split()[len(pattern.split())]) - -class Benchmark(object): - def __init__(self, name, runner, asc_good, units, - check=lambda:True, sizefactor=1): - if sizefactor > 1: - self.name = name + '*%d' % sizefactor - else: - self.name = name - self._basename = name - self._run = runner - self.asc_good = asc_good - self.units = units - self.check = check - self.sizefactor = sizefactor - def __mul__(self, n): - return Benchmark(self._basename, self._run, self.asc_good, self.units, - self.check, self.sizefactor * n) - def run(self, exe): - self.latest_output = '' - try: - result, latest_output = self._run(exe, self.sizefactor) - self.latest_output = latest_output - except BenchmarkFailed, e: - result = '-FAILED-' - return result - -def external_dependency(dirname, svnurl, revision=None): - directory = py.path.local(__file__).dirpath().join(dirname) - wc = py.path.svnwc(directory) - wc.checkout(svnurl, rev=revision) - return True - -def run_cmd(cmd): - pipe = os.popen(cmd + ' 2>&1') - r = pipe.read() - status = pipe.close() - if status: - raise BenchmarkFailed(status) - return r - -def run_pystone(executable, sizefactor=1): - from pypy.tool import autopath - distdir = py.path.local(autopath.pypydir).dirpath() - pystone = py.path.local(autopath.libpythondir).join('test', 'pystone.py') - txt = run_cmd('"%s" "%s" %d' % (executable, pystone, 50000 * sizefactor)) - return get_result(txt, PYSTONE_PATTERN), txt - -def run_richards(executable, sizefactor=1): - richards = py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py') - txt = run_cmd('"%s" %s %d' % (executable, richards, 5 * sizefactor)) - return get_result(txt, RICHARDS_PATTERN), txt - -def run_translate(executable): - translate = py.path.local(__file__).dirpath().dirpath().join('goal').join('translate.py') - target = py.path.local(__file__).dirpath().dirpath().join('goal').join('targetrpystonedalone.py') - argstr = '%s %s --batch --backendopt --no-compile %s > /dev/null 2> /dev/null' - T = time.time() - status = os.system(argstr%(executable, translate, target)) - r = time.time() - T - if status: - raise BenchmarkFailed(status) - return r - -def run_templess(executable, sizefactor=1): - """ run some script in the templess package - - templess is some simple templating language. - We have a copy at - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/templess' - """ - here = py.path.local(__file__).dirpath() - pypath = os.path.dirname(os.path.dirname(py.__file__)) - templessdir = here.join('templess') - testscript = templessdir.join('test/oneshot.py') - command = 'PYTHONPATH="%s:%s" "%s" "%s" %d' % (here, pypath, - executable, testscript, - 100 * sizefactor) - txt = run_cmd(command) - for line in txt.split('\n'): - if '.' in line: - try: - return float(line) / sizefactor, txt - except ValueError: - pass - else: - raise BenchmarkFailed - -def check_templess(): - return external_dependency('templess', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/templess') - -def run_gadfly(executable, sizefactor=1): - """ run some tests in the gadfly pure Python database """ - here = py.path.local(__file__).dirpath() - gadfly = here.join('gadfly') - testscript = gadfly.join('test', 'testsubset.py') - command = 'PYTHONPATH="%s" "%s" "%s" %d' % (gadfly, executable, testscript, - sizefactor) - txt = run_cmd(command) - return get_result(txt, 'Total running time:') / sizefactor, txt - -def check_gadfly(): - return external_dependency('gadfly', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/gadflyZip', - 70117) - -def run_mako(executable, sizefactor=1): - """ run some tests in the mako templating system """ - here = py.path.local(__file__).dirpath() - mako = here.join('mako') - testscript = mako.join('examples', 'bench', 'basic.py') - command = 'PYTHONPATH="%s" "%s" "%s" -n%d mako' % (mako.join('lib'), - executable, testscript, - 2000 * sizefactor) - txt = run_cmd(command) - return get_result(txt, 'Mako:'), txt - -def check_mako(): - return external_dependency('mako', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/mako', - 70118) - -def check_translate(): - return False # XXX what should we do about the dependency on ctypes? - -class LanguageShootoutBenchmark(Benchmark): - def __init__(self, name, sizefactor=1, test=False): - self.test = test - self.basename = name - Benchmark.__init__(self, name, self.runner, False, 'ms', - self.check, sizefactor) - - def __mul__(self, i): - return LanguageShootoutBenchmark(self.name, self.sizefactor * i, - self.test) - - def runner(self, executable, sizefactor=1): - shootout = py.path.local(__file__).dirpath().join( - 'shootout_benchmarks') - argsfile = shootout.join('tests.yml') - if self.test: - kind = 'test' - else: - kind = 'run' - args = yaml.load(argsfile.read())[self.basename][kind]['args'] - progname = str(shootout.join(self.basename)) + '.py' - cmd = 'time -f "%s" %s %s %s %d' % (TIME_FMT, executable, progname, - " ".join(args), sizefactor) - txt = run_cmd(cmd) - return get_result(txt, 'elapsed time:'), txt - - def check(self): - return external_dependency('shootout_benchmarks', - 'http://codespeak.net/svn/pypy/benchmarks/shootout') - -BENCHMARKS = [Benchmark('richards', run_richards, False, 'ms'), - Benchmark('pystone', run_pystone, True, ''), - Benchmark('translate', run_translate, False, 'ms', - check_translate), - Benchmark('templess', run_templess, False, - 's', check_templess), - Benchmark('gadfly2', run_gadfly, False, - 's', check_gadfly), - Benchmark('mako', run_mako, False, - 's', check_mako), - ] - -SHOOTOUT_NAMES = ['binary-trees', 'fannkuch', 'fasta', 'float', - 'meteor-contest', 'nbody', 'spectral-norm'] - -#for name in SHOOTOUT_NAMES: -# BENCHMARKS.append(LanguageShootoutBenchmark(name)) - -BENCHMARKS_BY_NAME = {} -for _b in BENCHMARKS: - BENCHMARKS_BY_NAME[_b.name] = _b --- a/pypy/translator/benchmark/result.py +++ /dev/null @@ -1,188 +0,0 @@ -import os, pickle, sys, time, re - -STAT2TITLE = { - 'stat:st_mtime': "date", - 'exe_name': "executable", -} - -def stat2title(s): - if s.startswith('bench:'): - return s[6:] - else: - return STAT2TITLE.get(s, s) - - -class BenchmarkResultSet(object): - def __init__(self, max_results=10): - self.benchmarks = {} - self.max_results = max_results - - def result(self, exe, allowcreate=False): - if exe in self.benchmarks or not allowcreate: - return self.benchmarks[exe] - else: - r = self.benchmarks[exe] = BenchmarkResult(exe, self.max_results) - return r - - def txt_summary(self, stats, **kw): - sortkey = kw.get('sortby', 'stat:st_mtime') - lst = self.benchmarks.values() - lst.sort(key=lambda x:x.getstat(sortkey, None), reverse=kw.get('reverse', False)) - if 'filteron' in kw: - filteron = kw['filteron'] - lst = [r for r in lst if filteron(r)] - relto = kw.get('relto', None) - table = [[(stat2title(s),0) for s in stats]] - for r in lst: - row = [] - for stat in stats: - if stat.startswith('bench:'): - benchname = stat[6:] - if r.getstat(stat, None) is None: - row.append(('XXX',-1)) - elif relto: - factor = self.result(relto).getstat(stat)/r.getstat(stat) - if not r.asc_goods[benchname]: - factor = 1/factor - s, f = r.fmtstat(stat) - row.append((s + ' (%6.2fx)'%factor, f)) - else: - row.append(r.fmtstat(stat)) - else: - row.append(r.fmtstat(stat)) - table.append(row) - widths = [0 for thing in stats] - for row in table: - for i, cell in enumerate(row): - widths[i] = max(len(cell[0]), widths[i]) - concretetable = [] - concreterow = [] - for w, cell in zip(widths, table[0]): - concreterow.append(cell[0].center(w)) - concretetable.append(' '.join(concreterow)) - for row in table[1:]: - concreterow = [] - for w, cell in zip(widths, row): - concreterow.append("%*s"%(cell[1]*w, cell[0])) - concretetable.append(' '.join(concreterow)) - return concretetable - -class BenchmarkResult(object): - IDS = {} - - def __init__(self, exe, max_results=10): - self.max_results = max_results - self.exe_stat = os.stat(exe) - self.exe_name = exe - self.codesize = os.popen('size "%s" | tail -n1 | cut -f1'%(exe,)).read().strip() - try: - self.pypy_rev = int(os.popen( - exe + ' -c "import sys; print sys.pypy_version_info[-1]" 2>/dev/null').read().strip()) - except ValueError: - self.pypy_rev = -1 - self.best_benchmarks = {} - self.benchmarks = {} - self.asc_goods = {} - self.run_counts = {} - - def run_benchmark(self, benchmark, verbose=False): - self.asc_goods[benchmark.name] = benchmark.asc_good - if self.run_counts.get(benchmark.name, 0) > self.max_results: - return -1 - print 'running', benchmark.name, 'for', self.exe_name, - if verbose and self.pypy_rev > 0: - print '[rev %d]' % self.pypy_rev, - sys.stdout.flush() - new_result = benchmark.run(self.exe_name) - print new_result - if verbose: - print '{' - lines = benchmark.latest_output.splitlines(False) - for line in lines[:80]: - print '\t' + line - if len(lines) > 80: - print '\t....' - print '}' - self.run_counts[benchmark.name] = self.run_counts.get(benchmark.name, 0) + 1 - if new_result == '-FAILED-': - return 0 - self.benchmarks.setdefault(benchmark.name, []).append(new_result) - if benchmark.name in self.best_benchmarks: - old_result = self.best_benchmarks[benchmark.name] - if benchmark.asc_good: - new_result = max(new_result, old_result) - else: - new_result = min(new_result, old_result) - self.best_benchmarks[benchmark.name] = new_result - return 1 - - def getstat(self, *args): - # oh for supplied-p! - return_default = False - if len(args) == 1: - stat, = args - else: - stat, default = args - return_default = True - if hasattr(self, stat): - return getattr(self, stat) - if stat == 'exe': - myid = len(BenchmarkResult.IDS) - myid = BenchmarkResult.IDS.setdefault(self, myid) - return '[%s]' % myid - statkind, statdetail = stat.split(':') - if statkind == 'stat': - return getattr(self.exe_stat, statdetail) - elif statkind == 'bench': - if return_default: - return self.best_benchmarks.get(statdetail, default) - else: - return self.best_benchmarks[statdetail] - else: - 1/0 - - def fmtstat(self, *args): - stat = args[0] - statvalue = self.getstat(*args) - if stat == 'stat:st_mtime': - return time.ctime(statvalue), -1 - elif stat == 'exe_name': - return os.path.basename(statvalue), -1 - elif stat.startswith('bench:'): - from pypy.translator.benchmark import benchmarks - statkind, statdetail = stat.split(':', 1) - if '*' in statdetail: - statdetail = statdetail.split('*')[0] - b = benchmarks.BENCHMARKS_BY_NAME[statdetail] - return "%8.2f%s"%(statvalue, b.units), 1 - elif stat == 'pypy_rev': - return str(statvalue), 1 - else: - return str(statvalue), -1 - - def summary(self, stats): - return [self.getstat(stat) for stat in stats] - - def is_stable(self, name): - try: - return self.n_results[name] >= self.max_results - except: - return False - -if __name__ == '__main__': - import autopath - from pypy.translator.benchmark import benchmarks, result - import cPickle - if os.path.exists('foo.pickle'): - s = cPickle.load(open('foo.pickle', 'rb')) - else: - s = result.BenchmarkResultSet(4) - for exe in sys.argv[1:]: - r = s.result(exe) - r.run_benchmark(benchmarks.BENCHMARKS_BY_NAME['richards']) - r.run_benchmark(benchmarks.BENCHMARKS_BY_NAME['pystone']) - cPickle.dump(s, open('foo.pickle', 'wb')) - stats = ['stat:st_mtime', 'exe_name', 'bench:richards', 'bench:pystone'] - - for row in s.txt_summary(stats, sortby="exe_name", reverse=True, relto="/usr/local/bin/python2.4"): - print row --- a/pypy/translator/benchmark/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -import py - -def pytest_ignore_collect(path): - return path.basename == "test" --- a/pypy/translator/benchmark/jitbench.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys, os -from optparse import OptionParser - -parser = OptionParser() -parser.add_option( - '--size-factor-list', dest='sizefactorlist', - default='1,2,5,20,1,2,5,20,1,2,5,20', - ) -options, args = parser.parse_args(sys.argv[1:]) -args = args or [sys.executable] -executables = [os.path.abspath(executable) for executable in args] -sizefactors = [int(s) for s in options.sizefactorlist.split(',')] - -os.chdir(os.path.dirname(sys.argv[0]) or '.') - -errors = [] - -for sizefactor in sizefactors: - for executable in executables: - sys.argv[1:] = [executable, '--pickle=jitbench.benchmark_result', - '-v', '--no-cpython', - '--size-factor=%d' % sizefactor] - try: - execfile('bench-custom.py') - except SystemExit, e: - errors.append('%s:*%s: %s' % (executable, sizefactor, e)) - -if errors: - print '\n'.join(errors) - sys.exit(1) From commits-noreply at bitbucket.org Tue Dec 14 11:41:54 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:54 -0600 (CST) Subject: [pypy-svn] pypy commit 076d21ac0a58: closed branch remove-sys-recursionlimit Message-ID: <20101214104154.34BAE241416@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User convert-repo # Date 1292309532 0 # Node ID 076d21ac0a585e46b7ee0709b9f49aef805e069c # Parent 9a0936e25c6f4aa5a4d0cfe30455ea29907e4d5a closed branch remove-sys-recursionlimit From commits-noreply at bitbucket.org Tue Dec 14 11:41:54 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:54 -0600 (CST) Subject: [pypy-svn] pypy commit 0e3099e3cc33: svn merge -r80002:80045 ../trunk Message-ID: <20101214104154.ED05624142A@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Amaury Forgeot d'Arc # Date 1292281506 0 # Node ID 0e3099e3cc33d9036ee057e45c1040add5299f97 # Parent 5385e710b33f751664f42473ecb7f498350fee66 svn merge -r80002:80045 ../trunk --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -103,9 +103,9 @@ class Arguments(object): make_sure_not_resized(self.keywords_w) make_sure_not_resized(self.arguments_w) - if w_stararg is not None and space.is_true(w_stararg): + if w_stararg is not None: self._combine_starargs_wrapped(w_stararg) - if w_starstararg is not None and space.is_true(w_starstararg): + if w_starstararg is not None: self._combine_starstarargs_wrapped(w_starstararg) # if we have a call where **args are used at the callsite # we shouldn't let the JIT see the argument matching --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -71,6 +71,8 @@ corresponding Unix manual entries for mo if hasattr(os, 'chown'): interpleveldefs['chown'] = 'interp_posix.chown' + if hasattr(os, 'lchown'): + interpleveldefs['lchown'] = 'interp_posix.lchown' if hasattr(os, 'ftruncate'): interpleveldefs['ftruncate'] = 'interp_posix.ftruncate' if hasattr(os, 'fsync'): @@ -86,6 +88,8 @@ corresponding Unix manual entries for mo if hasattr(os, 'kill') and sys.platform != 'win32': interpleveldefs['kill'] = 'interp_posix.kill' interpleveldefs['abort'] = 'interp_posix.abort' + if hasattr(os, 'killpg'): + interpleveldefs['killpg'] = 'interp_posix.killpg' if hasattr(os, 'getpid'): interpleveldefs['getpid'] = 'interp_posix.getpid' if hasattr(os, 'link'): @@ -117,6 +121,12 @@ corresponding Unix manual entries for mo interpleveldefs['ttyname'] = 'interp_posix.ttyname' if hasattr(os, 'getloadavg'): interpleveldefs['getloadavg'] = 'interp_posix.getloadavg' + if hasattr(os, 'mkfifo'): + interpleveldefs['mkfifo'] = 'interp_posix.mkfifo' + if hasattr(os, 'mknod'): + interpleveldefs['mknod'] = 'interp_posix.mknod' + if hasattr(os, 'nice'): + interpleveldefs['nice'] = 'interp_posix.nice' for name in ['setsid', 'getuid', 'geteuid', 'getgid', 'getegid', 'setuid', 'seteuid', 'setgid', 'setegid', 'getpgrp', 'setpgrp', --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -16,6 +16,7 @@ class TestW_DictObject: space = self.space d = self.space.newdict() assert not self.space.is_true(d) + assert d.r_dict_content is None def test_nonempty(self): space = self.space --- a/pypy/translator/c/src/stack.h +++ b/pypy/translator/c/src/stack.h @@ -11,27 +11,17 @@ * It is needed to have RPyThreadStaticTLS, too. */ #include "thread.h" +extern char *_LLstacktoobig_stack_start; + void LL_stack_unwind(void); -int LL_stack_too_big_slowpath(void); +char LL_stack_too_big_slowpath(long); /* returns 0 (ok) or 1 (too big) */ -extern volatile char *_LLstacktoobig_stack_base_pointer; -extern long _LLstacktoobig_stack_min; -extern long _LLstacktoobig_stack_max; +/* some macros referenced from pypy.rlib.rstack */ +#define OP_STACK_CURRENT(r) r = (long)&r +#define LL_stack_get_start() ((long)_LLstacktoobig_stack_start) +#define LL_stack_get_length() MAX_STACK_SIZE +#define LL_stack_get_start_adr() ((long)&_LLstacktoobig_stack_start) /* JIT */ -static int LL_stack_too_big(void) -{ - /* The fast path of stack_too_big, called extremely often. - Making it static makes an *inlinable* copy of this small - function's implementation in each compilation unit. */ - char local; - long diff = &local - _LLstacktoobig_stack_base_pointer; - /* common case: we are still in the same thread as last time - we checked, and still in the allowed part of the stack */ - return ((diff < _LLstacktoobig_stack_min || - diff > _LLstacktoobig_stack_max) - /* if not, call the slow path */ - && LL_stack_too_big_slowpath()); -} #ifdef __GNUC__ # define PYPY_INHIBIT_TAIL_CALL() asm("/* inhibit_tail_call */") @@ -61,68 +51,75 @@ long PYPY_NOINLINE _LL_stack_growing_dir return &local - parent; } -volatile char *_LLstacktoobig_stack_base_pointer = NULL; -long _LLstacktoobig_stack_min = 0; -long _LLstacktoobig_stack_max = 0; -RPyThreadStaticTLS _LLstacktoobig_stack_base_pointer_key; +char *_LLstacktoobig_stack_start = NULL; +int stack_direction = 0; +RPyThreadStaticTLS start_tls_key; -int LL_stack_too_big_slowpath(void) +char LL_stack_too_big_slowpath(long current) { - char local; long diff; - char *baseptr; - /* Check that the stack is less than MAX_STACK_SIZE bytes bigger - than the value recorded in stack_base_pointer. The base - pointer is updated to the current value if it is still NULL - or if we later find a &local that is below it. The real - stack base pointer is stored in thread-local storage, but we - try to minimize its overhead by keeping a local copy in - stack_pointer_pointer. */ + char *baseptr, *curptr = (char*)current; - if (_LLstacktoobig_stack_min == _LLstacktoobig_stack_max /* == 0 */) { + /* The stack_start variable is updated to match the current value + if it is still 0 or if we later find a 'curptr' position + that is below it. The real stack_start pointer is stored in + thread-local storage, but we try to minimize its overhead by + keeping a local copy in _LLstacktoobig_stack_start. */ + + if (stack_direction == 0) { /* not initialized */ /* XXX We assume that initialization is performed early, when there is still only one thread running. This allows us to ignore race conditions here */ - char *errmsg = RPyThreadStaticTLS_Create( - &_LLstacktoobig_stack_base_pointer_key); + char *errmsg = RPyThreadStaticTLS_Create(&start_tls_key); if (errmsg) { /* XXX should we exit the process? */ fprintf(stderr, "Internal PyPy error: %s\n", errmsg); return 1; } if (_LL_stack_growing_direction(NULL) > 0) - _LLstacktoobig_stack_max = MAX_STACK_SIZE; + stack_direction = +1; else - _LLstacktoobig_stack_min = -MAX_STACK_SIZE; + stack_direction = -1; } - baseptr = (char *) RPyThreadStaticTLS_Get( - _LLstacktoobig_stack_base_pointer_key); + baseptr = (char *) RPyThreadStaticTLS_Get(start_tls_key); if (baseptr != NULL) { - diff = &local - baseptr; - if (_LLstacktoobig_stack_min <= diff && - diff <= _LLstacktoobig_stack_max) { - /* within bounds */ - _LLstacktoobig_stack_base_pointer = baseptr; + diff = curptr - baseptr; + if (((unsigned long)diff) < (unsigned long)MAX_STACK_SIZE) { + /* within bounds, probably just had a thread switch */ + _LLstacktoobig_stack_start = baseptr; return 0; } - if ((_LLstacktoobig_stack_min == 0 && diff < 0) || - (_LLstacktoobig_stack_max == 0 && diff > 0)) { - /* we underflowed the stack, which means that - the initial estimation of the stack base must - be revised (see below) */ + if (stack_direction > 0) { + if (diff < 0 && diff > -MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } else { - return 1; /* stack overflow */ + if (diff >= MAX_STACK_SIZE && diff < 2*MAX_STACK_SIZE) + ; /* stack underflow */ + else + return 1; /* stack overflow (probably) */ } + /* else we underflowed the stack, which means that + the initial estimation of the stack base must + be revised */ } /* update the stack base pointer to the current value */ - baseptr = &local; - RPyThreadStaticTLS_Set(_LLstacktoobig_stack_base_pointer_key, baseptr); - _LLstacktoobig_stack_base_pointer = baseptr; + if (stack_direction > 0) { + /* the valid range is [curptr:curptr+MAX_STACK_SIZE] */ + baseptr = curptr; + } + else { + /* the valid range is [curptr-MAX_STACK_SIZE+1:curptr+1] */ + baseptr = curptr - MAX_STACK_SIZE + 1; + } + RPyThreadStaticTLS_Set(start_tls_key, baseptr); + _LLstacktoobig_stack_start = baseptr; return 0; } --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -56,9 +56,8 @@ class W_DictMultiObject(W_Object): else: if w_type is None: w_type = space.w_dict - w_self = space.allocate_instance(W_DictMultiObject, w_type) - W_DictMultiObject.__init__(w_self, space) - w_self.initialize_as_rdict() + w_self = space.allocate_instance(EmptyDictImplementation, w_type) + EmptyDictImplementation.__init__(w_self, space) return w_self def __init__(self, space): @@ -108,7 +107,7 @@ class W_DictMultiObject(W_Object): def impl_setitem_str(self, key, w_value): raise NotImplementedError("abstract base class") - def impl_setitem(self, w_key, w_value): + def impl_setitem(self, w_key, w_value): raise NotImplementedError("abstract base class") def impl_delitem(self, w_key): @@ -409,6 +408,45 @@ class WaryDictImplementation(StrDictImpl return self.shadowed[i] +class EmptyDictImplementation(W_DictMultiObject): + def __init__(self, space): + self.space = space + + def impl_setitem(self, w_key, w_value): + self._as_rdict().impl_fallback_setitem(w_key, w_value) + + def impl_setitem_str(self, key, w_value): + self._as_rdict().impl_fallback_setitem_str(key, w_value) + + def impl_delitem(self, w_key): + raise KeyError + + def impl_length(self): + return 0 + + def impl_getitem_str(self, key): + return None + + def impl_getitem(self, w_key): + # in case the key is unhashable, try to hash it + self.space.hash(w_key) + # return None anyway + return None + + def impl_iter(self): + # XXX I guess it's not important to be fast in this case? + return self._as_rdict().impl_fallback_iter() + + def impl_clear(self): + self.r_dict_content = None + + def _as_rdict(self): + r_dict_content = self.initialize_as_rdict() + return self + + def _clear_fields(self): + pass + class RDictIteratorImplementation(IteratorImplementation): def __init__(self, space, dictimplementation): IteratorImplementation.__init__(self, space, dictimplementation) --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -101,7 +101,6 @@ class X86FrameManager(FrameManager): return StackLoc(i, get_ebp_ofs(i), 1, box_type) class RegAlloc(object): - exc = False def __init__(self, assembler, translate_support_code=False): assert isinstance(translate_support_code, bool) @@ -428,7 +427,10 @@ class RegAlloc(object): locs = [self.loc(op.getarg(i)) for i in range(op.numargs())] locs_are_ref = [op.getarg(i).type == REF for i in range(op.numargs())] fail_index = self.assembler.cpu.get_fail_descr_number(op.getdescr()) - self.assembler.generate_failure(fail_index, locs, self.exc, + # note: no exception should currently be set in llop.get_exception_addr + # even if this finish may be an exit_frame_with_exception (in this case + # the exception instance is in locs[0]). + self.assembler.generate_failure(fail_index, locs, False, locs_are_ref) self.possibly_free_vars_for_op(op) --- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py +++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py @@ -3,6 +3,7 @@ from pypy.jit.backend.llsupport.asmmemmg from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper from pypy.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib import debug def test_get_index(): @@ -184,9 +185,11 @@ class TestAsmMemoryManager: def test_blockbuildermixin(translated=True): mc = BlockBuilderMixin(translated) + writtencode = [] for i in range(mc.SUBBLOCK_SIZE * 2 + 3): assert mc.get_relative_pos() == i mc.writechar(chr(i % 255)) + writtencode.append(chr(i % 255)) if translated: assert mc._cursubindex == 3 assert mc._cursubblock @@ -196,16 +199,26 @@ def test_blockbuildermixin(translated=Tr # for i in range(0, mc.SUBBLOCK_SIZE * 2 + 3, 2): mc.overwrite(i, chr((i + 63) % 255)) + writtencode[i] = chr((i + 63) % 255) # p = lltype.malloc(rffi.CCHARP.TO, mc.SUBBLOCK_SIZE * 2 + 3, flavor='raw') addr = rffi.cast(lltype.Signed, p) mc.copy_to_raw_memory(addr) # for i in range(mc.SUBBLOCK_SIZE * 2 + 3): - if i & 1: - assert p[i] == chr(i % 255) - else: - assert p[i] == chr((i + 63) % 255) + assert p[i] == writtencode[i] + # + debug._log = debug.DebugLog() + try: + mc._dump(addr, 'test-logname-section') + log = list(debug._log) + finally: + debug._log = None + encoded = ''.join(writtencode).encode('hex').upper() + ataddr = '@%x' % addr + assert log == [('test-logname-section', + [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])] + # lltype.free(p, flavor='raw') def test_blockbuildermixin2(): --- a/pypy/jit/backend/llsupport/asmmemmgr.py +++ b/pypy/jit/backend/llsupport/asmmemmgr.py @@ -2,6 +2,8 @@ import sys from pypy.rlib.rarithmetic import intmask, r_uint, LONG_BIT from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rmmap +from pypy.rlib.debug import debug_start, debug_print, debug_stop +from pypy.rlib.debug import have_debug_prints from pypy.rpython.lltypesystem import lltype, llmemory, rffi @@ -265,6 +267,31 @@ class BlockBuilderMixin(object): targetindex -= self.SUBBLOCK_SIZE assert not block + def _dump(self, addr, logname, backend=None): + debug_start(logname) + if have_debug_prints(): + # + if backend is not None: + debug_print('BACKEND', backend) + # + from pypy.jit.backend.hlinfo import highleveljitinfo + if highleveljitinfo.sys_executable: + debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable) + # + HEX = '0123456789ABCDEF' + dump = [] + src = rffi.cast(rffi.CCHARP, addr) + for p in range(self.get_relative_pos()): + o = ord(src[p]) + dump.append(HEX[o >> 4]) + dump.append(HEX[o & 15]) + debug_print('CODE_DUMP', + '@%x' % addr, + '+0 ', # backwards compatibility + ''.join(dump)) + # + debug_stop(logname) + def materialize(self, asmmemmgr, allblocks, gcrootmap=None): size = self.get_relative_pos() malloced = asmmemmgr.malloc(size, size) --- a/pypy/rpython/module/ll_os.py +++ b/pypy/rpython/module/ll_os.py @@ -99,7 +99,7 @@ class UnicodeTraits: return 'll_os.ll_os_w' + name def registering_str_unicode(posixfunc, condition=True): - if not condition: + if not condition or posixfunc is None: return registering(None, condition=False) func_name = posixfunc.__name__ @@ -1129,6 +1129,19 @@ class RegisterOs(BaseLazyRegistering): return extdef([str, int, int], None, "ll_os.ll_os_chown", llimpl=os_chown_llimpl) + @registering_if(os, 'lchown') + def register_os_lchown(self): + os_lchown = self.llexternal('lchown',[rffi.CCHARP, rffi.INT, rffi.INT], + rffi.INT) + + def os_lchown_llimpl(path, uid, gid): + res = os_lchown(path, uid, gid) + if res == -1: + raise OSError(rposix.get_errno(), "os_lchown failed") + + return extdef([str, int, int], None, "ll_os.ll_os_lchown", + llimpl=os_lchown_llimpl) + @registering_if(os, 'readlink') def register_os_readlink(self): os_readlink = self.llexternal('readlink', @@ -1323,6 +1336,33 @@ class RegisterOs(BaseLazyRegistering): return extdef([traits.str, traits.str], s_None, llimpl=rename_llimpl, export_name=traits.ll_os_name('rename')) + @registering_str_unicode(getattr(os, 'mkfifo', None)) + def register_os_mkfifo(self, traits): + os_mkfifo = self.llexternal(traits.posix_function_name('mkfifo'), + [traits.CCHARP, rffi.MODE_T], rffi.INT) + + def mkfifo_llimpl(path, mode): + res = rffi.cast(lltype.Signed, os_mkfifo(path, mode)) + if res < 0: + raise OSError(rposix.get_errno(), "os_mkfifo failed") + + return extdef([traits.str, int], s_None, llimpl=mkfifo_llimpl, + export_name=traits.ll_os_name('mkfifo')) + + @registering_str_unicode(getattr(os, 'mknod', None)) + def register_os_mknod(self, traits): + os_mknod = self.llexternal(traits.posix_function_name('mknod'), + [traits.CCHARP, rffi.MODE_T, rffi.INT], + rffi.INT) # xxx: actually ^^^ dev_t + + def mknod_llimpl(path, mode, dev): + res = rffi.cast(lltype.Signed, os_mknod(path, mode, dev)) + if res < 0: + raise OSError(rposix.get_errno(), "os_mknod failed") + + return extdef([traits.str, int, int], s_None, llimpl=mknod_llimpl, + export_name=traits.ll_os_name('mknod')) + @registering(os.umask) def register_os_umask(self): os_umask = self.llexternal(underscore_on_windows+'umask', [rffi.MODE_T], rffi.MODE_T) @@ -1348,6 +1388,20 @@ class RegisterOs(BaseLazyRegistering): return extdef([int, int], s_None, llimpl=kill_llimpl, export_name="ll_os.ll_os_kill") + @registering_if(os, 'killpg') + def register_os_killpg(self): + os_killpg = self.llexternal('killpg', [rffi.INT, rffi.INT], + rffi.INT) + + def killpg_llimpl(pid, sig): + res = rffi.cast(lltype.Signed, os_killpg(rffi.cast(rffi.INT, pid), + rffi.cast(rffi.INT, sig))) + if res < 0: + raise OSError(rposix.get_errno(), "os_killpg failed") + + return extdef([int, int], s_None, llimpl=killpg_llimpl, + export_name="ll_os.ll_os_killpg") + @registering_if(os, 'link') def register_os_link(self): os_link = self.llexternal('link', [rffi.CCHARP, rffi.CCHARP], @@ -1444,6 +1498,25 @@ class RegisterOs(BaseLazyRegistering): return extdef([int], s_None, llimpl=_exit_llimpl, export_name="ll_os.ll_os__exit") + @registering_if(os, 'nice') + def register_os_nice(self): + os_nice = self.llexternal('nice', [rffi.INT], rffi.INT) + + def nice_llimpl(inc): + # Assume that the system provides a standard-compliant version + # of nice() that returns the new priority. Nowadays, FreeBSD + # might be the last major non-compliant system (xxx check me). + rposix.set_errno(0) + res = rffi.cast(lltype.Signed, os_nice(inc)) + if res == -1: + err = rposix.get_errno() + if err != 0: + raise OSError(err, "os_nice failed") + return res + + return extdef([int], int, llimpl=nice_llimpl, + export_name="ll_os.ll_os_nice") + # --------------------------- os.stat & variants --------------------------- @registering(os.fstat) --- a/pypy/translator/c/test/test_extfunc.py +++ b/pypy/translator/c/test/test_extfunc.py @@ -4,6 +4,7 @@ import os, time, sys from pypy.tool.udir import udir from pypy.rlib.rarithmetic import r_longlong from pypy.translator.c.test.test_genc import compile +from pypy.translator.c.test.test_standalone import StandaloneTests posix = __import__(os.name) # note: clock synchronizes itself! @@ -404,6 +405,28 @@ def test_os_rename(): assert os.path.exists(tmpfile2) assert not os.path.exists(tmpfile1) +if hasattr(os, 'mkfifo'): + def test_os_mkfifo(): + tmpfile = str(udir.join('test_os_mkfifo.txt')) + def does_stuff(): + os.mkfifo(tmpfile, 0666) + f1 = compile(does_stuff, []) + f1() + import stat + st = os.lstat(tmpfile) + assert stat.S_ISFIFO(st.st_mode) + +if hasattr(os, 'mknod'): + def test_os_mknod(): + import stat + tmpfile = str(udir.join('test_os_mknod.txt')) + def does_stuff(): + os.mknod(tmpfile, 0600 | stat.S_IFIFO, 0) + f1 = compile(does_stuff, []) + f1() + st = os.lstat(tmpfile) + assert stat.S_ISFIFO(st.st_mode) + def test_os_umask(): def does_stuff(): mask1 = os.umask(0660) @@ -516,6 +539,62 @@ elif hasattr(os, 'waitpid'): # for what reason do they want us to shift by 8? See the doc assert status1 >> 8 == 4 +if hasattr(os, 'kill'): + def test_kill_to_send_sigusr1(): + import signal + from pypy.module.signal import interp_signal + def does_stuff(): + interp_signal.pypysig_setflag(signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR1) + interp_signal.pypysig_ignore(signal.SIGUSR1) + while True: + n = interp_signal.pypysig_poll() + if n < 0 or n == signal.SIGUSR1: + break + return n + f1 = compile(does_stuff, []) + got_signal = f1() + assert got_signal == signal.SIGUSR1 + +if hasattr(os, 'killpg'): + def test_killpg(): + import signal + from pypy.module.signal import interp_signal + def does_stuff(): + interp_signal.pypysig_setflag(signal.SIGUSR1) + os.killpg(os.getpgrp(), signal.SIGUSR1) + interp_signal.pypysig_ignore(signal.SIGUSR1) + while True: + n = interp_signal.pypysig_poll() + if n < 0 or n == signal.SIGUSR1: + break + return n + f1 = compile(does_stuff, []) + got_signal = f1() + assert got_signal == signal.SIGUSR1 + +if hasattr(os, 'chown') and hasattr(os, 'lchown'): + def test_os_chown_lchown(): + path1 = udir.join('test_os_chown_lchown-1.txt') + path2 = udir.join('test_os_chown_lchown-2.txt') + path1.write('foobar') + path2.mksymlinkto('some-broken-symlink') + tmpfile1 = str(path1) + tmpfile2 = str(path2) + def does_stuff(): + # xxx not really a test, just checks that they are callable + os.chown(tmpfile1, os.getuid(), os.getgid()) + os.lchown(tmpfile1, os.getuid(), os.getgid()) + os.lchown(tmpfile2, os.getuid(), os.getgid()) + try: + os.chown(tmpfile2, os.getuid(), os.getgid()) + except OSError: + pass + else: + raise AssertionError("os.chown(broken symlink) should raise") + f1 = compile(does_stuff, []) + f1() + # ____________________________________________________________ def _real_getenv(var): @@ -783,3 +862,19 @@ if hasattr(os, 'fchdir'): finally: os.chdir(localdir) assert res == True + +# ____________________________________________________________ + + +class TestExtFuncStandalone(StandaloneTests): + + if hasattr(os, 'nice'): + def test_os_nice(self): + def does_stuff(argv): + res = os.nice(3) + print 'os.nice returned', res + return 0 + t, cbuilder = self.compile(does_stuff) + data = cbuilder.cmdexec('') + res = os.nice(0) + 3 + assert data.startswith('os.nice returned %d\n' % res) --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -98,6 +98,39 @@ def get_subversion_info(space): space.wrap(svnbranch), space.wrap(str(svn_revision()))]) +def get_mercurial_info(space): + '''Obtain Mercurial version information by invoking the 'hg' command.''' + # TODO: support extracting from .hg_archival.txt + import py + from subprocess import Popen, PIPE + + pypyroot = os.path.abspath(os.path.join(pypydir, '..')) + hgexe = py.path.local.sysfind('hg') + + if hgexe and os.path.isdir(os.path.join(pypyroot, '.hg')): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull + + p = Popen([str(hgexe), 'id', '-i', pypyroot], stdout=PIPE, env=env) + hgid = p.stdout.read().strip() + + p = Popen([str(hgexe), 'id', '-t', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip().split()[0] + + if hgtag == 'tip': + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip() + + return space.newtuple([space.wrap('PyPy'), + space.wrap(hgtag), + space.wrap(hgid)]) + else: + return space.w_None + def tuple2hex(ver): d = {'alpha': 0xA, 'beta': 0xB, --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -161,6 +161,7 @@ def ftruncate(space, fd, length): ftruncate.unwrap_spec = [ObjSpace, "c_int", r_longlong] def fsync(space, w_fd): + """Force write of file with filedescriptor to disk.""" fd = space.c_filedescriptor_w(w_fd) try: os.fsync(fd) @@ -169,6 +170,8 @@ def fsync(space, w_fd): fsync.unwrap_spec = [ObjSpace, W_Root] def fdatasync(space, w_fd): + """Force write of file with filedescriptor to disk. +Does not force update of metadata.""" fd = space.c_filedescriptor_w(w_fd) try: os.fdatasync(fd) @@ -177,6 +180,8 @@ def fdatasync(space, w_fd): fdatasync.unwrap_spec = [ObjSpace, W_Root] def fchdir(space, w_fd): + """Change to the directory of the given file descriptor. fildes must be +opened on a directory, not a file.""" fd = space.c_filedescriptor_w(w_fd) try: os.fchdir(fd) @@ -557,6 +562,27 @@ def rename(space, w_old, w_new): raise wrap_oserror(space, e) rename.unwrap_spec = [ObjSpace, W_Root, W_Root] +def mkfifo(space, w_filename, mode=0666): + """Create a FIFO (a POSIX named pipe).""" + try: + dispatch_filename(rposix.mkfifo)(space, w_filename, mode) + except OSError, e: + raise wrap_oserror2(space, e, w_filename) +mkfifo.unwrap_spec = [ObjSpace, W_Root, "c_int"] + +def mknod(space, w_filename, mode=0600, device=0): + """Create a filesystem node (file, device special file or named pipe) +named filename. mode specifies both the permissions to use and the +type of node to be created, being combined (bitwise OR) with one of +S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. For S_IFCHR and S_IFBLK, +device defines the newly created device special file (probably using +os.makedev()), otherwise it is ignored.""" + try: + dispatch_filename(rposix.mknod)(space, w_filename, mode, device) + except OSError, e: + raise wrap_oserror2(space, e, w_filename) +mknod.unwrap_spec = [ObjSpace, W_Root, "c_int", "c_int"] + def umask(space, mask): "Set the current numeric umask and return the previous umask." prevmask = os.umask(mask) @@ -580,6 +606,14 @@ def kill(space, pid, sig): raise wrap_oserror(space, e) kill.unwrap_spec = [ObjSpace, "c_int", "c_int"] +def killpg(space, pgid, sig): + "Kill a process group with a signal." + try: + os.killpg(pgid, sig) + except OSError, e: + raise wrap_oserror(space, e) +killpg.unwrap_spec = [ObjSpace, "c_int", "c_int"] + def abort(space): """Abort the interpreter immediately. This 'dumps core' or otherwise fails in the hardest way possible on the hosting operating system.""" @@ -1034,6 +1068,14 @@ def chown(space, path, uid, gid): return space.w_None chown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] +def lchown(space, path, uid, gid): + try: + os.lchown(path, uid, gid) + except OSError, e: + raise wrap_oserror(space, e, path) + return space.w_None +lchown.unwrap_spec = [ObjSpace, str, "c_nonnegint", "c_nonnegint"] + def getloadavg(space): try: load = os.getloadavg() @@ -1045,6 +1087,15 @@ def getloadavg(space): space.wrap(load[2])]) getloadavg.unwrap_spec = [ObjSpace] +def nice(space, inc): + "Decrease the priority of process by inc and return the new priority." + try: + res = os.nice(inc) + except OSError, e: + raise wrap_oserror(space, e) + return space.wrap(res) +nice.unwrap_spec = [ObjSpace, "c_int"] + if _WIN: from pypy.rlib import rwin32 --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1294,6 +1294,10 @@ class MetaInterpStaticData(object): num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) # + tokens = self.loop_tokens_exit_frame_with_exception_ref + num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) + self.cpu.exit_frame_with_exception_v = num + # self.globaldata = MetaInterpGlobalData(self) def _setup_once(self): --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -672,6 +672,67 @@ class AppTestPosix: f.close() os.chown(self.path, os.getuid(), os.getgid()) + if hasattr(os, 'lchown'): + def test_lchown(self): + os = self.posix + os.unlink(self.path) + raises(OSError, os.lchown, self.path, os.getuid(), os.getgid()) + os.symlink('foobar', self.path) + os.lchown(self.path, os.getuid(), os.getgid()) + + if hasattr(os, 'mkfifo'): + def test_mkfifo(self): + os = self.posix + os.mkfifo(self.path2 + 'test_mkfifo', 0666) + st = os.lstat(self.path2 + 'test_mkfifo') + import stat + assert stat.S_ISFIFO(st.st_mode) + + if hasattr(os, 'mknod'): + def test_mknod(self): + import stat + os = self.posix + # not very useful: os.mknod() without specifying 'mode' + os.mknod(self.path2 + 'test_mknod-1') + st = os.lstat(self.path2 + 'test_mknod-1') + assert stat.S_ISREG(st.st_mode) + # os.mknod() with S_IFIFO + os.mknod(self.path2 + 'test_mknod-2', 0600 | stat.S_IFIFO) + st = os.lstat(self.path2 + 'test_mknod-2') + assert stat.S_ISFIFO(st.st_mode) + + def test_mknod_with_ifchr(self): + # os.mknod() with S_IFCHR + # -- usually requires root priviledges -- + os = self.posix + if hasattr(os.lstat('.'), 'st_rdev'): + import stat + try: + os.mknod(self.path2 + 'test_mknod-3', 0600 | stat.S_IFCHR, + 0x105) + except OSError, e: + skip("os.mknod() with S_IFCHR: got %r" % (e,)) + else: + st = os.lstat(self.path2 + 'test_mknod-3') + assert stat.S_ISCHR(st.st_mode) + assert st.st_rdev == 0x105 + + if hasattr(os, 'nice') and hasattr(os, 'fork') and hasattr(os, 'waitpid'): + def test_nice(self): + os = self.posix + myprio = os.nice(0) + # + pid = os.fork() + if pid == 0: # in the child + res = os.nice(3) + os._exit(res) + # + pid1, status1 = os.waitpid(pid, 0) + assert pid1 == pid + assert os.WIFEXITED(status1) + assert os.WEXITSTATUS(status1) == myprio + 3 + + class AppTestEnvironment(object): def setup_class(cls): cls.space = space --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -64,6 +64,7 @@ class Module(MixedModule): 'pypy_version_info' : 'version.get_pypy_version_info(space)', 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', + '_mercurial' : 'version.get_mercurial_info(space)', 'hexversion' : 'version.get_hexversion(space)', 'displayhook' : 'hook.displayhook', --- a/pypy/jit/backend/x86/rx86.py +++ b/pypy/jit/backend/x86/rx86.py @@ -347,6 +347,8 @@ def common_modes(group): INSN_rr = insn(rex_w, chr(base+1), register(2,8), register(1,1), '\xC0') INSN_br = insn(rex_w, chr(base+1), register(2,8), stack_bp(1)) INSN_rb = insn(rex_w, chr(base+3), register(1,8), stack_bp(2)) + INSN_rm = insn(rex_w, chr(base+3), register(1,8), mem_reg_plus_const(2)) + INSN_rj = insn(rex_w, chr(base+3), register(1,8), '\x05', immediate(2)) INSN_bi8 = insn(rex_w, '\x83', orbyte(base), stack_bp(1), immediate(2,'b')) INSN_bi32= insn(rex_w, '\x81', orbyte(base), stack_bp(1), immediate(2)) @@ -364,7 +366,7 @@ def common_modes(group): INSN_bi32(mc, offset, immed) INSN_bi._always_inline_ = True # try to constant-fold single_byte() - return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br + return INSN_ri, INSN_rr, INSN_rb, INSN_bi, INSN_br, INSN_rm, INSN_rj def select_8_or_32_bit_immed(insn_8, insn_32): def INSN(*args): @@ -442,24 +444,22 @@ class AbstractX86CodeBuilder(object): # ------------------------------ Arithmetic ------------------------------ - ADD_ri, ADD_rr, ADD_rb, _, _ = common_modes(0) - OR_ri, OR_rr, OR_rb, _, _ = common_modes(1) - AND_ri, AND_rr, AND_rb, _, _ = common_modes(4) - SUB_ri, SUB_rr, SUB_rb, _, _ = common_modes(5) - XOR_ri, XOR_rr, XOR_rb, _, _ = common_modes(6) - CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br = common_modes(7) + ADD_ri, ADD_rr, ADD_rb, _, _, ADD_rm, ADD_rj = common_modes(0) + OR_ri, OR_rr, OR_rb, _, _, OR_rm, OR_rj = common_modes(1) + AND_ri, AND_rr, AND_rb, _, _, AND_rm, AND_rj = common_modes(4) + SUB_ri, SUB_rr, SUB_rb, _, _, SUB_rm, SUB_rj = common_modes(5) + XOR_ri, XOR_rr, XOR_rb, _, _, XOR_rm, XOR_rj = common_modes(6) + CMP_ri, CMP_rr, CMP_rb, CMP_bi, CMP_br, CMP_rm, CMP_rj = common_modes(7) CMP_mi8 = insn(rex_w, '\x83', orbyte(7<<3), mem_reg_plus_const(1), immediate(2, 'b')) CMP_mi32 = insn(rex_w, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) CMP_mi = select_8_or_32_bit_immed(CMP_mi8, CMP_mi32) - CMP_rm = insn(rex_w, '\x3B', register(1, 8), mem_reg_plus_const(2)) CMP_mr = insn(rex_w, '\x39', register(2, 8), mem_reg_plus_const(1)) CMP_ji8 = insn(rex_w, '\x83', '\x3D', immediate(1), immediate(2, 'b')) CMP_ji32 = insn(rex_w, '\x81', '\x3D', immediate(1), immediate(2)) CMP_ji = select_8_or_32_bit_immed(CMP_ji8, CMP_ji32) - CMP_rj = insn(rex_w, '\x3B', register(1, 8), '\x05', immediate(2)) CMP32_mi = insn(rex_nw, '\x81', orbyte(7<<3), mem_reg_plus_const(1), immediate(2)) --- a/pypy/rlib/rstack.py +++ b/pypy/rlib/rstack.py @@ -6,21 +6,22 @@ RPython-compliant way, intended mostly f import inspect from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.rarithmetic import r_uint +from pypy.rlib import rgc from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rpython.lltypesystem import rffi, lltype +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.controllerentry import Controller, SomeControlledInstance from pypy.translator.tool.cbuild import ExternalCompilationInfo def stack_unwind(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_unwind(lltype.Void) raise RuntimeError("cannot unwind stack in non-translated versions") def stack_capture(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop ptr = llop.stack_capture(OPAQUE_STATE_HEADER_PTR) return frame_stack_top_controller.box(ptr) raise RuntimeError("cannot unwind stack in non-translated versions") @@ -28,26 +29,57 @@ def stack_capture(): def stack_frames_depth(): if we_are_translated(): - from pypy.rpython.lltypesystem.lloperation import llop return llop.stack_frames_depth(lltype.Signed) else: return len(inspect.stack()) +# ____________________________________________________________ + compilation_info = ExternalCompilationInfo(includes=['src/stack.h']) -stack_too_big = rffi.llexternal('LL_stack_too_big', [], rffi.INT, - compilation_info=compilation_info, - _nowrapper=True, - _callable=lambda: _zero, - sandboxsafe=True) -_zero = rffi.cast(rffi.INT, 0) +def llexternal(name, args, res): + return rffi.llexternal(name, args, res, compilation_info=compilation_info, + sandboxsafe=True, _nowrapper=True) + +_stack_get_start = llexternal('LL_stack_get_start', [], lltype.Signed) +_stack_get_length = llexternal('LL_stack_get_length', [], lltype.Signed) +_stack_too_big_slowpath = llexternal('LL_stack_too_big_slowpath', + [lltype.Signed], lltype.Char) +# the following is used by the JIT +_stack_get_start_adr = llexternal('LL_stack_get_start_adr', [], lltype.Signed) + def stack_check(): - if rffi.cast(lltype.Signed, stack_too_big()): + if not we_are_translated(): + return + # + # Load the "current" stack position, or at least some address that + # points close to the current stack head + current = llop.stack_current(lltype.Signed) + # + # Load these variables from C code + start = _stack_get_start() + length = _stack_get_length() + # + # Common case: if 'current' is within [start:start+length], everything + # is fine + ofs = r_uint(current - start) + if ofs < r_uint(length): + return + # + # Else call the slow path + stack_check_slowpath(current) +stack_check._always_inline_ = True + + at rgc.no_collect +def stack_check_slowpath(current): + if ord(_stack_too_big_slowpath(current)): + # Now we are sure that the stack is really too big. Note that the # stack_unwind implementation is different depending on if stackless # is enabled. If it is it unwinds the stack, otherwise it simply # raises a RuntimeError. stack_unwind() +stack_check_slowpath._dont_inline_ = True # ____________________________________________________________ --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -115,6 +115,7 @@ class AbstractLLCPU(AbstractCPU): self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = lambda: (0, 0, 0) def _setup_exception_handling_translated(self): @@ -138,9 +139,20 @@ class AbstractLLCPU(AbstractCPU): # in the assignment to self.saved_exc_value, as needed. self.saved_exc_value = exc_value + from pypy.rlib import rstack + STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def insert_stack_check(): + startaddr = rstack._stack_get_start_adr() + length = rstack._stack_get_length() + f = llhelper(STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + slowpathaddr = rffi.cast(lltype.Signed, f) + return startaddr, length, slowpathaddr + self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.save_exception = save_exception + self.insert_stack_check = insert_stack_check def _setup_on_leave_jitted_untranslated(self): # assume we don't need a backend leave in this case --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -528,6 +528,8 @@ LL_OPERATIONS = { 'get_stack_depth_limit':LLOp(sideeffects=False), 'set_stack_depth_limit':LLOp(), + 'stack_current': LLOp(sideeffects=False), + # __________ misc operations __________ 'keepalive': LLOp(), --- a/pypy/jit/backend/x86/test/test_ztranslation.py +++ b/pypy/jit/backend/x86/test/test_ztranslation.py @@ -93,6 +93,9 @@ class TestTranslationX86(CCompiledMixin) assert res == expected def test_direct_assembler_call_translates(self): + """Test CALL_ASSEMBLER and the recursion limit""" + from pypy.rlib.rstackovf import StackOverflow + class Thing(object): def __init__(self, val): self.val = val @@ -135,9 +138,35 @@ class TestTranslationX86(CCompiledMixin) i += 1 return frame.thing.val - res = self.meta_interp(main, [0], inline=True, + driver2 = JitDriver(greens = [], reds = ['n']) + + def main2(bound): + try: + while portal2(bound) == -bound+1: + bound *= 2 + except StackOverflow: + pass + return bound + + def portal2(n): + while True: + driver2.jit_merge_point(n=n) + n -= 1 + if n <= 0: + return n + n = portal2(n) + assert portal2(10) == -9 + + def mainall(codeno, bound): + return main(codeno) + main2(bound) + + res = self.meta_interp(mainall, [0, 1], inline=True, policy=StopAtXPolicy(change)) - assert res == main(0) + print hex(res) + assert res & 255 == main(0) + bound = res & ~255 + assert 1024 <= bound <= 131072 + assert bound & (bound-1) == 0 # a power of two class TestTranslationRemoveTypePtrX86(CCompiledMixin): --- a/pypy/jit/backend/x86/codebuf.py +++ b/pypy/jit/backend/x86/codebuf.py @@ -10,8 +10,10 @@ from pypy.jit.backend.x86 import valgrin # like this if IS_X86_32: codebuilder_cls = X86_32_CodeBuilder + backend_name = 'x86' elif IS_X86_64: codebuilder_cls = X86_64_CodeBuilder + backend_name = 'x86_64' class MachineCodeBlockWrapper(BlockBuilderMixin, @@ -34,3 +36,4 @@ class MachineCodeBlockWrapper(BlockBuild adr = rffi.cast(rffi.LONGP, p - WORD) adr[0] = intmask(adr[0] - p) valgrind.discard_translations(addr, self.get_relative_pos()) + self._dump(addr, "jit-backend-dump", backend_name) --- a/pypy/jit/backend/model.py +++ b/pypy/jit/backend/model.py @@ -8,6 +8,7 @@ class AbstractCPU(object): done_with_this_frame_int_v = -1 done_with_this_frame_ref_v = -1 done_with_this_frame_float_v = -1 + exit_frame_with_exception_v = -1 total_compiled_loops = 0 total_compiled_bridges = 0 total_freed_loops = 0 --- a/pypy/rlib/rposix.py +++ b/pypy/rlib/rposix.py @@ -135,6 +135,20 @@ def rmdir(path): else: return os.rmdir(path.as_bytes()) + at specialize.argtype(0) +def mkfifo(path, mode): + if isinstance(path, str): + os.mkfifo(path, mode) + else: + os.mkfifo(path.as_bytes(), mode) + + at specialize.argtype(0) +def mknod(path, mode, device): + if isinstance(path, str): + os.mknod(path, mode, device) + else: + os.mknod(path.as_bytes(), mode, device) + if os.name == 'nt': import nt def _getfullpathname(path): --- a/pypy/jit/backend/x86/tool/viewcode.py +++ b/pypy/jit/backend/x86/tool/viewcode.py @@ -1,11 +1,11 @@ #! /usr/bin/env python """ -Viewer for the CODE_DUMP output of compiled programs generating code. +Viewer for the output of compiled programs generating code. +Use on the log files created with 'PYPYLOG=jit-backend-dump:log'. Try: - ./viewcode.py dumpfile.txt -or - /tmp/usession-xxx/testing_1/testing_1 -var 4 2>&1 | ./viewcode.py + ./viewcode.py --text log # text only disassembly + ./viewcode.py log # also includes a pygame viewer """ import autopath @@ -179,6 +179,7 @@ class World(object): self.symbols = {} self.logentries = {} self.backend_name = None + self.executable_name = None def parse(self, f, textonly=True): for line in f: @@ -214,7 +215,9 @@ class World(object): self.logentries[addr] = pieces[3] elif line.startswith('SYS_EXECUTABLE '): filename = line[len('SYS_EXECUTABLE '):].strip() - self.symbols.update(load_symbols(filename)) + if filename != self.executable_name: + self.symbols.update(load_symbols(filename)) + self.executable_name = filename def find_cross_references(self): # find cross-references between blocks @@ -375,10 +378,19 @@ if __name__ == '__main__': showgraph = False else: showgraph = True - if len(sys.argv) == 1: - f = sys.stdin - else: - f = open(sys.argv[1], 'r') + if len(sys.argv) != 2: + print >> sys.stderr, __doc__ + sys.exit(2) + # + import cStringIO + from pypy.tool import logparser + log1 = logparser.parse_log_file(sys.argv[1]) + text1 = logparser.extract_category(log1, catprefix='jit-backend-dump') + f = cStringIO.StringIO() + f.writelines(text1) + f.seek(0) + del log1, text1 + # world = World() world.parse(f) if showgraph: --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -84,6 +84,7 @@ class Assembler386(object): self.fail_boxes_count = 0 self._current_depths_cache = (0, 0) self.datablockwrapper = None + self.stack_check_slowpath_imm = imm0 self.teardown() def leave_jitted_hook(self): @@ -122,6 +123,7 @@ class Assembler386(object): self._build_float_constants() if hasattr(gc_ll_descr, 'get_malloc_fixedsize_slowpath_addr'): self._build_malloc_fixedsize_slowpath() + self._build_stack_check_slowpath() debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') @@ -194,6 +196,82 @@ class Assembler386(object): rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.malloc_fixedsize_slowpath2 = rawstart + _STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], + lltype.Void)) + def _build_stack_check_slowpath(self): + from pypy.rlib import rstack + mc = codebuf.MachineCodeBlockWrapper() + mc.PUSH_r(ebp.value) + mc.MOV_rr(ebp.value, esp.value) + # + if IS_X86_64: + # on the x86_64, we have to save all the registers that may + # have been used to pass arguments + for reg in [edi, esi, edx, ecx, r8, r9]: + mc.PUSH_r(reg.value) + mc.SUB_ri(esp.value, 8*8) + for i in range(8): + mc.MOVSD_sx(8*i, i) # xmm0 to xmm7 + # + if IS_X86_32: + mc.LEA_rb(eax.value, +8) + mc.PUSH_r(eax.value) + elif IS_X86_64: + mc.LEA_rb(edi.value, +16) + mc.AND_ri(esp.value, -16) + # + f = llhelper(self._STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) + addr = rffi.cast(lltype.Signed, f) + mc.CALL(imm(addr)) + # + mc.MOV(eax, heap(self.cpu.pos_exception())) + mc.TEST_rr(eax.value, eax.value) + mc.J_il8(rx86.Conditions['NZ'], 0) + jnz_location = mc.get_relative_pos() + # + if IS_X86_64: + # restore the registers + for i in range(7, -1, -1): + mc.MOVSD_xs(i, 8*i) + for i, reg in [(6, r9), (5, r8), (4, ecx), + (3, edx), (2, esi), (1, edi)]: + mc.MOV_rb(reg.value, -8*i) + # + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.RET() + # + # patch the JNZ above + offset = mc.get_relative_pos() - jnz_location + assert 0 < offset <= 127 + mc.overwrite(jnz_location-1, chr(offset)) + # clear the exception from the global position + mc.MOV(eax, heap(self.cpu.pos_exc_value())) + mc.MOV(heap(self.cpu.pos_exception()), imm0) + mc.MOV(heap(self.cpu.pos_exc_value()), imm0) + # save the current exception instance into fail_boxes_ptr[0] + adr = self.fail_boxes_ptr.get_addr_for_num(0) + mc.MOV(heap(adr), eax) + # call the helper function to set the GC flag on the fail_boxes_ptr + # array (note that there is no exception any more here) + addr = self.cpu.get_on_leave_jitted_int(save_exception=False) + mc.CALL(imm(addr)) + # + assert self.cpu.exit_frame_with_exception_v >= 0 + mc.MOV_ri(eax.value, self.cpu.exit_frame_with_exception_v) + # + # footer -- note the ADD, which skips the return address of this + # function, and will instead return to the caller's caller. Note + # also that we completely ignore the saved arguments, because we + # are interrupting the function. + mc.MOV_rr(esp.value, ebp.value) + mc.POP_r(ebp.value) + mc.ADD_ri(esp.value, WORD) + mc.RET() + # + rawstart = mc.materialize(self.cpu.asmmemmgr, []) + self.stack_check_slowpath_imm = imm(rawstart) + def assemble_loop(self, inputargs, operations, looptoken, log): '''adds the following attributes to looptoken: _x86_loop_code (an integer giving an address) @@ -468,6 +546,24 @@ class Assembler386(object): for regloc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(regloc.value) + def _call_header_with_stack_check(self): + startaddr, length, slowpathaddr = self.cpu.insert_stack_check() + if slowpathaddr == 0: + pass # no stack check (e.g. not translated) + else: + self.mc.MOV(eax, esp) # MOV eax, current + self.mc.SUB(eax, heap(startaddr)) # SUB eax, [startaddr] + self.mc.CMP(eax, imm(length)) # CMP eax, length + self.mc.J_il8(rx86.Conditions['B'], 0) # JB .skip + jb_location = self.mc.get_relative_pos() + self.mc.CALL(self.stack_check_slowpath_imm) # CALL slowpath + # patch the JB above # .skip: + offset = self.mc.get_relative_pos() - jb_location + assert 0 < offset <= 127 + self.mc.overwrite(jb_location-1, chr(offset)) + # + self._call_header() + def _call_footer(self): self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) @@ -485,7 +581,7 @@ class Assembler386(object): # XXX this can be improved greatly. Right now it'll behave like # a normal call nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) for i in range(len(nonfloatlocs)): loc = nonfloatlocs[i] @@ -526,7 +622,7 @@ class Assembler386(object): unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] nonfloatlocs, floatlocs = arglocs - self._call_header() + self._call_header_with_stack_check() self.mc.LEA_rb(esp.value, self._get_offset_of_ebp_from_esp(stackdepth)) # The lists are padded with Nones --- /dev/null +++ b/.hgignore @@ -0,0 +1,53 @@ +syntax:glob +*.py[co] + +testresult +site-packages +pypy/module/cpyext/src/*.o +pypy/bin/pypy-c +pypy/translator/jvm/src/pypy/*.class +pypy/module/_stackless/test/ +pypy/module/cpyext/test/*.errors +pypy/doc/*.html +pypy/doc/basicblock.asc +pypy/doc/*.svninfo +pypy/translator/jvm/.project +pypy/translator/jvm/.classpath +pypy/translator/jvm/eclipse-bin +pypy/translator/benchmark/docutils +pypy/translator/benchmark/templess +pypy/translator/benchmark/gadfly +pypy/translator/benchmark/mako +pypy/translator/benchmark/bench-custom.benchmark_result +pypy/translator/benchmark/shootout_benchmarks +pypy/module/_stackless/ +pypy/translator/goal/pypy-translation-snapshot +pypy/translator/goal/pypy-c* +pypy/translator/goal/*.exe +pypy/translator/goal/target*-c +pypy/_cache +site-packages/*.egg +site-packages/*.pth +pypy/doc/statistic/*.html +pypy/doc/statistic/*.eps +pypy/doc/statistic/*.pdf +pypy/translator/cli/src/pypylib.dll +pypy/translator/cli/src/query.exe +pypy/translator/cli/src/main.exe +lib_pypy/ctypes_config_cache/_*_cache.py +lib_pypy/ctypes_config_cache/_*_*_.py +pypy/translator/cli/query-descriptions +pypy/doc/discussion/*.html +pypy/doc/discussion/ +include/*.h +include/*.inl +pypy/doc/config/*.html +pypy/doc/config/style.css +pypy/doc/config/ +pypy/doc/jit/*.html +pypy/doc/jit/style.css +pypy/doc/image/lattice1.png +pypy/doc/image/lattice2.png +pypy/doc/image/lattice3.png +pypy/doc/image/stackless_informal.png +pypy/doc/image/parsing_example*.png From commits-noreply at bitbucket.org Tue Dec 14 11:41:55 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:55 -0600 (CST) Subject: [pypy-svn] pypy commit 151684b7ac9d: I don't think we use that for anything any more. Delete (correct me if I'm wrong) Message-ID: <20101214104155.8B44D241316@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Maciej Fijalkowski # Date 1292309657 0 # Node ID 151684b7ac9d50b2f5b86ed90715a02867e9a490 # Parent 74bbe61137d921345adafcb80d004ce7d8860ace I don't think we use that for anything any more. Delete (correct me if I'm wrong) --- a/pypy/translator/benchmark/autopath.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') -import py # note: py is imported only AFTER the path has been set -libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) -libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) - -if __name__ == '__main__': - __clone() --- a/pypy/translator/benchmark/bench-custom.py +++ /dev/null @@ -1,127 +0,0 @@ -# benchmarks on a unix machine. - -import autopath -from pypy.translator.benchmark.result import BenchmarkResultSet -from pypy.translator.benchmark.benchmarks import BENCHMARKS -import os, sys, time, pickle, re, py - -SPLIT_TABLE = True # useful when executable names are very long - -def get_executables(args): #sorted by revision number (highest first) - exes = sorted(args, key=os.path.getmtime) - r = [] - for exe in exes: - if '/' not in exe: - r.append('./' + exe) - else: - r.append(exe) - return r - -def main(options, args): - if os.path.exists(options.picklefile): - benchmark_result = pickle.load(open(options.picklefile, 'rb')) - else: - benchmark_result = BenchmarkResultSet() - - benchmarks = [] - for b in BENCHMARKS: - if b.name in options.benchmarks: - if not b.check(): - print "can't run %s benchmark for some reason"%(b.name,) - else: - if int(options.sizefactor) > 1: - b = b * int(options.sizefactor) - benchmarks.append(b) - - exes = get_executables(args) - pythons = 'python2.6 python2.5 python2.4'.split() - full_pythons = [] - for python in pythons: - full_python = py.path.local.sysfind(python) - if full_python: - full_pythons.append(str(full_python)) - - sys.stdout.flush() - - refs = {} - final_error_count = 0 - - if not options.nocpython: - exes = full_pythons + exes - - for i in range(int(options.runcount)) or [None]: - if i is not None: - for exe in exes: - for b in benchmarks: - br = benchmark_result.result(exe, allowcreate=True) - result = br.run_benchmark(b, verbose=options.verbose) - if not result: - final_error_count += 1 - - if options.relto: - relto = options.relto - else: - relto = full_pythons[0] - if relto not in benchmark_result.benchmarks: - continue - - pickle.dump(benchmark_result, open(options.picklefile, 'wb')) - - exe_stats = ['stat:st_mtime', 'exe_name', 'pypy_rev'] - if not SPLIT_TABLE: - stats = exe_stats[:] - else: - stats = ['exe'] - for b in benchmarks: - stats.append('bench:'+b.name) - kwds = {'relto': relto, - 'filteron' :lambda r: r.exe_name in exes, - } - for row in benchmark_result.txt_summary(stats, **kwds): - print row - if SPLIT_TABLE: - print - print 'Reference:' - for row in benchmark_result.txt_summary(['exe'] + exe_stats, - **kwds): - print row - print - - if final_error_count: - raise SystemExit("%d benchmark run(s) failed (see -FAILED- above)" - % final_error_count) - -if __name__ == '__main__': - from optparse import OptionParser - parser = OptionParser() - default_benches = ','.join([b.name for b in BENCHMARKS if b.check()]) - parser.add_option( - '--benchmarks', dest='benchmarks', - default=default_benches, - ) - parser.add_option( - '--pickle', dest='picklefile', - default='bench-custom.benchmark_result' - ) - parser.add_option( - '--runcount', dest='runcount', - default='1', - ) - parser.add_option( - '--relto', dest='relto', - default=None, - ) - parser.add_option( - '-v', '--verbose', action='store_true', dest='verbose', - default=None, - ) - parser.add_option( - '--no-cpython', action='store_true', dest='nocpython', - default=None, - ) - parser.add_option( - '--size-factor', dest='sizefactor', - default='1', - ) - options, args = parser.parse_args(sys.argv[1:]) - main(options, args) --- a/pypy/translator/benchmark/__init__.py +++ /dev/null @@ -1,1 +0,0 @@ -# --- a/pypy/translator/benchmark/benchmarks.py +++ /dev/null @@ -1,197 +0,0 @@ -import os, sys, time, pickle, re, py -import yaml - -class BenchmarkFailed(Exception): - pass - -PYSTONE_CMD = 'from test import pystone;pystone.main(%s)' -PYSTONE_PATTERN = 'This machine benchmarks at' - -RICHARDS_CMD = 'from richards import *;main(iterations=%d)' -RICHARDS_PATTERN = 'Average time per iteration:' - -TIME_FMT = 'max mem used: %Mk\nelapsed time: %e\nsystem time: %S\nuser time: %U\nCPU use: %P' - -def get_result(txt, pattern): - for line in txt.split('\n'): - if line.startswith(pattern): - break - else: - raise BenchmarkFailed - return float(line.split()[len(pattern.split())]) - -class Benchmark(object): - def __init__(self, name, runner, asc_good, units, - check=lambda:True, sizefactor=1): - if sizefactor > 1: - self.name = name + '*%d' % sizefactor - else: - self.name = name - self._basename = name - self._run = runner - self.asc_good = asc_good - self.units = units - self.check = check - self.sizefactor = sizefactor - def __mul__(self, n): - return Benchmark(self._basename, self._run, self.asc_good, self.units, - self.check, self.sizefactor * n) - def run(self, exe): - self.latest_output = '' - try: - result, latest_output = self._run(exe, self.sizefactor) - self.latest_output = latest_output - except BenchmarkFailed, e: - result = '-FAILED-' - return result - -def external_dependency(dirname, svnurl, revision=None): - directory = py.path.local(__file__).dirpath().join(dirname) - wc = py.path.svnwc(directory) - wc.checkout(svnurl, rev=revision) - return True - -def run_cmd(cmd): - pipe = os.popen(cmd + ' 2>&1') - r = pipe.read() - status = pipe.close() - if status: - raise BenchmarkFailed(status) - return r - -def run_pystone(executable, sizefactor=1): - from pypy.tool import autopath - distdir = py.path.local(autopath.pypydir).dirpath() - pystone = py.path.local(autopath.libpythondir).join('test', 'pystone.py') - txt = run_cmd('"%s" "%s" %d' % (executable, pystone, 50000 * sizefactor)) - return get_result(txt, PYSTONE_PATTERN), txt - -def run_richards(executable, sizefactor=1): - richards = py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py') - txt = run_cmd('"%s" %s %d' % (executable, richards, 5 * sizefactor)) - return get_result(txt, RICHARDS_PATTERN), txt - -def run_translate(executable): - translate = py.path.local(__file__).dirpath().dirpath().join('goal').join('translate.py') - target = py.path.local(__file__).dirpath().dirpath().join('goal').join('targetrpystonedalone.py') - argstr = '%s %s --batch --backendopt --no-compile %s > /dev/null 2> /dev/null' - T = time.time() - status = os.system(argstr%(executable, translate, target)) - r = time.time() - T - if status: - raise BenchmarkFailed(status) - return r - -def run_templess(executable, sizefactor=1): - """ run some script in the templess package - - templess is some simple templating language. - We have a copy at - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/templess' - """ - here = py.path.local(__file__).dirpath() - pypath = os.path.dirname(os.path.dirname(py.__file__)) - templessdir = here.join('templess') - testscript = templessdir.join('test/oneshot.py') - command = 'PYTHONPATH="%s:%s" "%s" "%s" %d' % (here, pypath, - executable, testscript, - 100 * sizefactor) - txt = run_cmd(command) - for line in txt.split('\n'): - if '.' in line: - try: - return float(line) / sizefactor, txt - except ValueError: - pass - else: - raise BenchmarkFailed - -def check_templess(): - return external_dependency('templess', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/templess') - -def run_gadfly(executable, sizefactor=1): - """ run some tests in the gadfly pure Python database """ - here = py.path.local(__file__).dirpath() - gadfly = here.join('gadfly') - testscript = gadfly.join('test', 'testsubset.py') - command = 'PYTHONPATH="%s" "%s" "%s" %d' % (gadfly, executable, testscript, - sizefactor) - txt = run_cmd(command) - return get_result(txt, 'Total running time:') / sizefactor, txt - -def check_gadfly(): - return external_dependency('gadfly', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/gadflyZip', - 70117) - -def run_mako(executable, sizefactor=1): - """ run some tests in the mako templating system """ - here = py.path.local(__file__).dirpath() - mako = here.join('mako') - testscript = mako.join('examples', 'bench', 'basic.py') - command = 'PYTHONPATH="%s" "%s" "%s" -n%d mako' % (mako.join('lib'), - executable, testscript, - 2000 * sizefactor) - txt = run_cmd(command) - return get_result(txt, 'Mako:'), txt - -def check_mako(): - return external_dependency('mako', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/mako', - 70118) - -def check_translate(): - return False # XXX what should we do about the dependency on ctypes? - -class LanguageShootoutBenchmark(Benchmark): - def __init__(self, name, sizefactor=1, test=False): - self.test = test - self.basename = name - Benchmark.__init__(self, name, self.runner, False, 'ms', - self.check, sizefactor) - - def __mul__(self, i): - return LanguageShootoutBenchmark(self.name, self.sizefactor * i, - self.test) - - def runner(self, executable, sizefactor=1): - shootout = py.path.local(__file__).dirpath().join( - 'shootout_benchmarks') - argsfile = shootout.join('tests.yml') - if self.test: - kind = 'test' - else: - kind = 'run' - args = yaml.load(argsfile.read())[self.basename][kind]['args'] - progname = str(shootout.join(self.basename)) + '.py' - cmd = 'time -f "%s" %s %s %s %d' % (TIME_FMT, executable, progname, - " ".join(args), sizefactor) - txt = run_cmd(cmd) - return get_result(txt, 'elapsed time:'), txt - - def check(self): - return external_dependency('shootout_benchmarks', - 'http://codespeak.net/svn/pypy/benchmarks/shootout') - -BENCHMARKS = [Benchmark('richards', run_richards, False, 'ms'), - Benchmark('pystone', run_pystone, True, ''), - Benchmark('translate', run_translate, False, 'ms', - check_translate), - Benchmark('templess', run_templess, False, - 's', check_templess), - Benchmark('gadfly2', run_gadfly, False, - 's', check_gadfly), - Benchmark('mako', run_mako, False, - 's', check_mako), - ] - -SHOOTOUT_NAMES = ['binary-trees', 'fannkuch', 'fasta', 'float', - 'meteor-contest', 'nbody', 'spectral-norm'] - -#for name in SHOOTOUT_NAMES: -# BENCHMARKS.append(LanguageShootoutBenchmark(name)) - -BENCHMARKS_BY_NAME = {} -for _b in BENCHMARKS: - BENCHMARKS_BY_NAME[_b.name] = _b --- a/pypy/translator/benchmark/result.py +++ /dev/null @@ -1,188 +0,0 @@ -import os, pickle, sys, time, re - -STAT2TITLE = { - 'stat:st_mtime': "date", - 'exe_name': "executable", -} - -def stat2title(s): - if s.startswith('bench:'): - return s[6:] - else: - return STAT2TITLE.get(s, s) - - -class BenchmarkResultSet(object): - def __init__(self, max_results=10): - self.benchmarks = {} - self.max_results = max_results - - def result(self, exe, allowcreate=False): - if exe in self.benchmarks or not allowcreate: - return self.benchmarks[exe] - else: - r = self.benchmarks[exe] = BenchmarkResult(exe, self.max_results) - return r - - def txt_summary(self, stats, **kw): - sortkey = kw.get('sortby', 'stat:st_mtime') - lst = self.benchmarks.values() - lst.sort(key=lambda x:x.getstat(sortkey, None), reverse=kw.get('reverse', False)) - if 'filteron' in kw: - filteron = kw['filteron'] - lst = [r for r in lst if filteron(r)] - relto = kw.get('relto', None) - table = [[(stat2title(s),0) for s in stats]] - for r in lst: - row = [] - for stat in stats: - if stat.startswith('bench:'): - benchname = stat[6:] - if r.getstat(stat, None) is None: - row.append(('XXX',-1)) - elif relto: - factor = self.result(relto).getstat(stat)/r.getstat(stat) - if not r.asc_goods[benchname]: - factor = 1/factor - s, f = r.fmtstat(stat) - row.append((s + ' (%6.2fx)'%factor, f)) - else: - row.append(r.fmtstat(stat)) - else: - row.append(r.fmtstat(stat)) - table.append(row) - widths = [0 for thing in stats] - for row in table: - for i, cell in enumerate(row): - widths[i] = max(len(cell[0]), widths[i]) - concretetable = [] - concreterow = [] - for w, cell in zip(widths, table[0]): - concreterow.append(cell[0].center(w)) - concretetable.append(' '.join(concreterow)) - for row in table[1:]: - concreterow = [] - for w, cell in zip(widths, row): - concreterow.append("%*s"%(cell[1]*w, cell[0])) - concretetable.append(' '.join(concreterow)) - return concretetable - -class BenchmarkResult(object): - IDS = {} - - def __init__(self, exe, max_results=10): - self.max_results = max_results - self.exe_stat = os.stat(exe) - self.exe_name = exe - self.codesize = os.popen('size "%s" | tail -n1 | cut -f1'%(exe,)).read().strip() - try: - self.pypy_rev = int(os.popen( - exe + ' -c "import sys; print sys.pypy_version_info[-1]" 2>/dev/null').read().strip()) - except ValueError: - self.pypy_rev = -1 - self.best_benchmarks = {} - self.benchmarks = {} - self.asc_goods = {} - self.run_counts = {} - - def run_benchmark(self, benchmark, verbose=False): - self.asc_goods[benchmark.name] = benchmark.asc_good - if self.run_counts.get(benchmark.name, 0) > self.max_results: - return -1 - print 'running', benchmark.name, 'for', self.exe_name, - if verbose and self.pypy_rev > 0: - print '[rev %d]' % self.pypy_rev, - sys.stdout.flush() - new_result = benchmark.run(self.exe_name) - print new_result - if verbose: - print '{' - lines = benchmark.latest_output.splitlines(False) - for line in lines[:80]: - print '\t' + line - if len(lines) > 80: - print '\t....' - print '}' - self.run_counts[benchmark.name] = self.run_counts.get(benchmark.name, 0) + 1 - if new_result == '-FAILED-': - return 0 - self.benchmarks.setdefault(benchmark.name, []).append(new_result) - if benchmark.name in self.best_benchmarks: - old_result = self.best_benchmarks[benchmark.name] - if benchmark.asc_good: - new_result = max(new_result, old_result) - else: - new_result = min(new_result, old_result) - self.best_benchmarks[benchmark.name] = new_result - return 1 - - def getstat(self, *args): - # oh for supplied-p! - return_default = False - if len(args) == 1: - stat, = args - else: - stat, default = args - return_default = True - if hasattr(self, stat): - return getattr(self, stat) - if stat == 'exe': - myid = len(BenchmarkResult.IDS) - myid = BenchmarkResult.IDS.setdefault(self, myid) - return '[%s]' % myid - statkind, statdetail = stat.split(':') - if statkind == 'stat': - return getattr(self.exe_stat, statdetail) - elif statkind == 'bench': - if return_default: - return self.best_benchmarks.get(statdetail, default) - else: - return self.best_benchmarks[statdetail] - else: - 1/0 - - def fmtstat(self, *args): - stat = args[0] - statvalue = self.getstat(*args) - if stat == 'stat:st_mtime': - return time.ctime(statvalue), -1 - elif stat == 'exe_name': - return os.path.basename(statvalue), -1 - elif stat.startswith('bench:'): - from pypy.translator.benchmark import benchmarks - statkind, statdetail = stat.split(':', 1) - if '*' in statdetail: - statdetail = statdetail.split('*')[0] - b = benchmarks.BENCHMARKS_BY_NAME[statdetail] - return "%8.2f%s"%(statvalue, b.units), 1 - elif stat == 'pypy_rev': - return str(statvalue), 1 - else: - return str(statvalue), -1 - - def summary(self, stats): - return [self.getstat(stat) for stat in stats] - - def is_stable(self, name): - try: - return self.n_results[name] >= self.max_results - except: - return False - -if __name__ == '__main__': - import autopath - from pypy.translator.benchmark import benchmarks, result - import cPickle - if os.path.exists('foo.pickle'): - s = cPickle.load(open('foo.pickle', 'rb')) - else: - s = result.BenchmarkResultSet(4) - for exe in sys.argv[1:]: - r = s.result(exe) - r.run_benchmark(benchmarks.BENCHMARKS_BY_NAME['richards']) - r.run_benchmark(benchmarks.BENCHMARKS_BY_NAME['pystone']) - cPickle.dump(s, open('foo.pickle', 'wb')) - stats = ['stat:st_mtime', 'exe_name', 'bench:richards', 'bench:pystone'] - - for row in s.txt_summary(stats, sortby="exe_name", reverse=True, relto="/usr/local/bin/python2.4"): - print row --- a/pypy/translator/benchmark/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -import py - -def pytest_ignore_collect(path): - return path.basename == "test" --- a/pypy/translator/benchmark/jitbench.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys, os -from optparse import OptionParser - -parser = OptionParser() -parser.add_option( - '--size-factor-list', dest='sizefactorlist', - default='1,2,5,20,1,2,5,20,1,2,5,20', - ) -options, args = parser.parse_args(sys.argv[1:]) -args = args or [sys.executable] -executables = [os.path.abspath(executable) for executable in args] -sizefactors = [int(s) for s in options.sizefactorlist.split(',')] - -os.chdir(os.path.dirname(sys.argv[0]) or '.') - -errors = [] - -for sizefactor in sizefactors: - for executable in executables: - sys.argv[1:] = [executable, '--pickle=jitbench.benchmark_result', - '-v', '--no-cpython', - '--size-factor=%d' % sizefactor] - try: - execfile('bench-custom.py') - except SystemExit, e: - errors.append('%s:*%s: %s' % (executable, sizefactor, e)) - -if errors: - print '\n'.join(errors) - sys.exit(1) From commits-noreply at bitbucket.org Tue Dec 14 11:41:56 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:41:56 -0600 (CST) Subject: [pypy-svn] pypy commit 53e05faa128c: Make c_filedescriptor_w more closely mirror CPython's PyObject_AsFileDescriptor (in behavior and structure), which doesn't bother verifying the int fits in 32-bits. Message-ID: <20101214104156.233D3241416@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Alex Gaynor # Date 1292317513 0 # Node ID 53e05faa128ca5a08b1dd95f706226694fc7245b # Parent 151684b7ac9d50b2f5b86ed90715a02867e9a490 Make c_filedescriptor_w more closely mirror CPython's PyObject_AsFileDescriptor (in behavior and structure), which doesn't bother verifying the int fits in 32-bits. --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -797,8 +797,8 @@ class ObjSpace(object): def call_obj_args(self, w_callable, w_obj, args): if not self.config.objspace.disable_call_speedhacks: - # XXX start of hack for performance - from pypy.interpreter.function import Function + # XXX start of hack for performance + from pypy.interpreter.function import Function if isinstance(w_callable, Function): return w_callable.call_obj_args(w_obj, args) # XXX end of hack for performance @@ -1184,24 +1184,27 @@ class ObjSpace(object): return value def c_filedescriptor_w(self, w_fd): - try: - fd = self.c_int_w(w_fd) - except OperationError, e: - if not e.match(self, self.w_TypeError): - raise + if (not self.isinstance_w(w_fd, self.w_int) and + not self.isinstance_w(w_fd, self.w_long)): try: - w_fileno = self.getattr(w_fd, self.wrap('fileno')) + w_fileno = self.getattr(w_fd, self.wrap("fileno")) except OperationError, e: if e.match(self, self.w_AttributeError): raise OperationError(self.w_TypeError, - self.wrap("argument must be an int, " - "or have a fileno() method.")) + self.wrap("argument must be an int, or have a fileno() " + "method.") + ) raise w_fd = self.call_function(w_fileno) - fd = self.c_int_w(w_fd) + if not self.isinstance_w(w_fd, self.w_int): + raise OperationError(self.w_TypeError, + self.wrap("fileno() must return an integer") + ) + fd = self.int_w(w_fd) if fd < 0: raise operationerrfmt(self.w_ValueError, - "file descriptor cannot be a negative integer (%d)", fd) + "file descriptor cannot be a negative integer (%d)", fd + ) return fd def warn(self, msg, w_warningcls): @@ -1403,4 +1406,3 @@ ObjSpace.IrregularOpTable = [ 'call_args', 'marshal_w', ] - From commits-noreply at bitbucket.org Tue Dec 14 11:47:22 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:47:22 -0600 (CST) Subject: [pypy-svn] pypy commit e45b4f1a2e50: Merge heads Message-ID: <20101214104722.45E4E1E106D@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292323561 -3600 # Node ID e45b4f1a2e50ef1b44a602033a2dd2b297bc144f # Parent 594f56f4cede7c763dca05d1ac0693f40a9b76d0 # Parent f05147cddb7176830601931fc023a556842a35ca Merge heads From commits-noreply at bitbucket.org Tue Dec 14 11:47:23 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:47:23 -0600 (CST) Subject: [pypy-svn] pypy commit 58f56bdf5c4b: Merge heads Message-ID: <20101214104723.30E871E1071@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292323601 -3600 # Node ID 58f56bdf5c4bf4ca311b0f6a794ed8a79a1a85a8 # Parent e45b4f1a2e50ef1b44a602033a2dd2b297bc144f # Parent 53e05faa128ca5a08b1dd95f706226694fc7245b Merge heads --- a/pypy/translator/benchmark/autopath.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') -import py # note: py is imported only AFTER the path has been set -libpythondir = str(py.path.local(pypydir).dirpath().join('lib-python', '2.5.2')) -libpythonmodifieddir = str(py.path.local(libpythondir).dirpath().join('modified-2.5.2')) - -if __name__ == '__main__': - __clone() --- a/pypy/translator/benchmark/bench-custom.py +++ /dev/null @@ -1,127 +0,0 @@ -# benchmarks on a unix machine. - -import autopath -from pypy.translator.benchmark.result import BenchmarkResultSet -from pypy.translator.benchmark.benchmarks import BENCHMARKS -import os, sys, time, pickle, re, py - -SPLIT_TABLE = True # useful when executable names are very long - -def get_executables(args): #sorted by revision number (highest first) - exes = sorted(args, key=os.path.getmtime) - r = [] - for exe in exes: - if '/' not in exe: - r.append('./' + exe) - else: - r.append(exe) - return r - -def main(options, args): - if os.path.exists(options.picklefile): - benchmark_result = pickle.load(open(options.picklefile, 'rb')) - else: - benchmark_result = BenchmarkResultSet() - - benchmarks = [] - for b in BENCHMARKS: - if b.name in options.benchmarks: - if not b.check(): - print "can't run %s benchmark for some reason"%(b.name,) - else: - if int(options.sizefactor) > 1: - b = b * int(options.sizefactor) - benchmarks.append(b) - - exes = get_executables(args) - pythons = 'python2.6 python2.5 python2.4'.split() - full_pythons = [] - for python in pythons: - full_python = py.path.local.sysfind(python) - if full_python: - full_pythons.append(str(full_python)) - - sys.stdout.flush() - - refs = {} - final_error_count = 0 - - if not options.nocpython: - exes = full_pythons + exes - - for i in range(int(options.runcount)) or [None]: - if i is not None: - for exe in exes: - for b in benchmarks: - br = benchmark_result.result(exe, allowcreate=True) - result = br.run_benchmark(b, verbose=options.verbose) - if not result: - final_error_count += 1 - - if options.relto: - relto = options.relto - else: - relto = full_pythons[0] - if relto not in benchmark_result.benchmarks: - continue - - pickle.dump(benchmark_result, open(options.picklefile, 'wb')) - - exe_stats = ['stat:st_mtime', 'exe_name', 'pypy_rev'] - if not SPLIT_TABLE: - stats = exe_stats[:] - else: - stats = ['exe'] - for b in benchmarks: - stats.append('bench:'+b.name) - kwds = {'relto': relto, - 'filteron' :lambda r: r.exe_name in exes, - } - for row in benchmark_result.txt_summary(stats, **kwds): - print row - if SPLIT_TABLE: - print - print 'Reference:' - for row in benchmark_result.txt_summary(['exe'] + exe_stats, - **kwds): - print row - print - - if final_error_count: - raise SystemExit("%d benchmark run(s) failed (see -FAILED- above)" - % final_error_count) - -if __name__ == '__main__': - from optparse import OptionParser - parser = OptionParser() - default_benches = ','.join([b.name for b in BENCHMARKS if b.check()]) - parser.add_option( - '--benchmarks', dest='benchmarks', - default=default_benches, - ) - parser.add_option( - '--pickle', dest='picklefile', - default='bench-custom.benchmark_result' - ) - parser.add_option( - '--runcount', dest='runcount', - default='1', - ) - parser.add_option( - '--relto', dest='relto', - default=None, - ) - parser.add_option( - '-v', '--verbose', action='store_true', dest='verbose', - default=None, - ) - parser.add_option( - '--no-cpython', action='store_true', dest='nocpython', - default=None, - ) - parser.add_option( - '--size-factor', dest='sizefactor', - default='1', - ) - options, args = parser.parse_args(sys.argv[1:]) - main(options, args) --- a/pypy/translator/benchmark/__init__.py +++ /dev/null @@ -1,1 +0,0 @@ -# --- a/pypy/translator/benchmark/benchmarks.py +++ /dev/null @@ -1,197 +0,0 @@ -import os, sys, time, pickle, re, py -import yaml - -class BenchmarkFailed(Exception): - pass - -PYSTONE_CMD = 'from test import pystone;pystone.main(%s)' -PYSTONE_PATTERN = 'This machine benchmarks at' - -RICHARDS_CMD = 'from richards import *;main(iterations=%d)' -RICHARDS_PATTERN = 'Average time per iteration:' - -TIME_FMT = 'max mem used: %Mk\nelapsed time: %e\nsystem time: %S\nuser time: %U\nCPU use: %P' - -def get_result(txt, pattern): - for line in txt.split('\n'): - if line.startswith(pattern): - break - else: - raise BenchmarkFailed - return float(line.split()[len(pattern.split())]) - -class Benchmark(object): - def __init__(self, name, runner, asc_good, units, - check=lambda:True, sizefactor=1): - if sizefactor > 1: - self.name = name + '*%d' % sizefactor - else: - self.name = name - self._basename = name - self._run = runner - self.asc_good = asc_good - self.units = units - self.check = check - self.sizefactor = sizefactor - def __mul__(self, n): - return Benchmark(self._basename, self._run, self.asc_good, self.units, - self.check, self.sizefactor * n) - def run(self, exe): - self.latest_output = '' - try: - result, latest_output = self._run(exe, self.sizefactor) - self.latest_output = latest_output - except BenchmarkFailed, e: - result = '-FAILED-' - return result - -def external_dependency(dirname, svnurl, revision=None): - directory = py.path.local(__file__).dirpath().join(dirname) - wc = py.path.svnwc(directory) - wc.checkout(svnurl, rev=revision) - return True - -def run_cmd(cmd): - pipe = os.popen(cmd + ' 2>&1') - r = pipe.read() - status = pipe.close() - if status: - raise BenchmarkFailed(status) - return r - -def run_pystone(executable, sizefactor=1): - from pypy.tool import autopath - distdir = py.path.local(autopath.pypydir).dirpath() - pystone = py.path.local(autopath.libpythondir).join('test', 'pystone.py') - txt = run_cmd('"%s" "%s" %d' % (executable, pystone, 50000 * sizefactor)) - return get_result(txt, PYSTONE_PATTERN), txt - -def run_richards(executable, sizefactor=1): - richards = py.path.local(__file__).dirpath().dirpath().join('goal').join('richards.py') - txt = run_cmd('"%s" %s %d' % (executable, richards, 5 * sizefactor)) - return get_result(txt, RICHARDS_PATTERN), txt - -def run_translate(executable): - translate = py.path.local(__file__).dirpath().dirpath().join('goal').join('translate.py') - target = py.path.local(__file__).dirpath().dirpath().join('goal').join('targetrpystonedalone.py') - argstr = '%s %s --batch --backendopt --no-compile %s > /dev/null 2> /dev/null' - T = time.time() - status = os.system(argstr%(executable, translate, target)) - r = time.time() - T - if status: - raise BenchmarkFailed(status) - return r - -def run_templess(executable, sizefactor=1): - """ run some script in the templess package - - templess is some simple templating language. - We have a copy at - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/templess' - """ - here = py.path.local(__file__).dirpath() - pypath = os.path.dirname(os.path.dirname(py.__file__)) - templessdir = here.join('templess') - testscript = templessdir.join('test/oneshot.py') - command = 'PYTHONPATH="%s:%s" "%s" "%s" %d' % (here, pypath, - executable, testscript, - 100 * sizefactor) - txt = run_cmd(command) - for line in txt.split('\n'): - if '.' in line: - try: - return float(line) / sizefactor, txt - except ValueError: - pass - else: - raise BenchmarkFailed - -def check_templess(): - return external_dependency('templess', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/templess') - -def run_gadfly(executable, sizefactor=1): - """ run some tests in the gadfly pure Python database """ - here = py.path.local(__file__).dirpath() - gadfly = here.join('gadfly') - testscript = gadfly.join('test', 'testsubset.py') - command = 'PYTHONPATH="%s" "%s" "%s" %d' % (gadfly, executable, testscript, - sizefactor) - txt = run_cmd(command) - return get_result(txt, 'Total running time:') / sizefactor, txt - -def check_gadfly(): - return external_dependency('gadfly', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/gadflyZip', - 70117) - -def run_mako(executable, sizefactor=1): - """ run some tests in the mako templating system """ - here = py.path.local(__file__).dirpath() - mako = here.join('mako') - testscript = mako.join('examples', 'bench', 'basic.py') - command = 'PYTHONPATH="%s" "%s" "%s" -n%d mako' % (mako.join('lib'), - executable, testscript, - 2000 * sizefactor) - txt = run_cmd(command) - return get_result(txt, 'Mako:'), txt - -def check_mako(): - return external_dependency('mako', - 'http://codespeak.net/svn/user/arigo/hack/pypy-hack/mako', - 70118) - -def check_translate(): - return False # XXX what should we do about the dependency on ctypes? - -class LanguageShootoutBenchmark(Benchmark): - def __init__(self, name, sizefactor=1, test=False): - self.test = test - self.basename = name - Benchmark.__init__(self, name, self.runner, False, 'ms', - self.check, sizefactor) - - def __mul__(self, i): - return LanguageShootoutBenchmark(self.name, self.sizefactor * i, - self.test) - - def runner(self, executable, sizefactor=1): - shootout = py.path.local(__file__).dirpath().join( - 'shootout_benchmarks') - argsfile = shootout.join('tests.yml') - if self.test: - kind = 'test' - else: - kind = 'run' - args = yaml.load(argsfile.read())[self.basename][kind]['args'] - progname = str(shootout.join(self.basename)) + '.py' - cmd = 'time -f "%s" %s %s %s %d' % (TIME_FMT, executable, progname, - " ".join(args), sizefactor) - txt = run_cmd(cmd) - return get_result(txt, 'elapsed time:'), txt - - def check(self): - return external_dependency('shootout_benchmarks', - 'http://codespeak.net/svn/pypy/benchmarks/shootout') - -BENCHMARKS = [Benchmark('richards', run_richards, False, 'ms'), - Benchmark('pystone', run_pystone, True, ''), - Benchmark('translate', run_translate, False, 'ms', - check_translate), - Benchmark('templess', run_templess, False, - 's', check_templess), - Benchmark('gadfly2', run_gadfly, False, - 's', check_gadfly), - Benchmark('mako', run_mako, False, - 's', check_mako), - ] - -SHOOTOUT_NAMES = ['binary-trees', 'fannkuch', 'fasta', 'float', - 'meteor-contest', 'nbody', 'spectral-norm'] - -#for name in SHOOTOUT_NAMES: -# BENCHMARKS.append(LanguageShootoutBenchmark(name)) - -BENCHMARKS_BY_NAME = {} -for _b in BENCHMARKS: - BENCHMARKS_BY_NAME[_b.name] = _b --- a/pypy/translator/benchmark/result.py +++ /dev/null @@ -1,188 +0,0 @@ -import os, pickle, sys, time, re - -STAT2TITLE = { - 'stat:st_mtime': "date", - 'exe_name': "executable", -} - -def stat2title(s): - if s.startswith('bench:'): - return s[6:] - else: - return STAT2TITLE.get(s, s) - - -class BenchmarkResultSet(object): - def __init__(self, max_results=10): - self.benchmarks = {} - self.max_results = max_results - - def result(self, exe, allowcreate=False): - if exe in self.benchmarks or not allowcreate: - return self.benchmarks[exe] - else: - r = self.benchmarks[exe] = BenchmarkResult(exe, self.max_results) - return r - - def txt_summary(self, stats, **kw): - sortkey = kw.get('sortby', 'stat:st_mtime') - lst = self.benchmarks.values() - lst.sort(key=lambda x:x.getstat(sortkey, None), reverse=kw.get('reverse', False)) - if 'filteron' in kw: - filteron = kw['filteron'] - lst = [r for r in lst if filteron(r)] - relto = kw.get('relto', None) - table = [[(stat2title(s),0) for s in stats]] - for r in lst: - row = [] - for stat in stats: - if stat.startswith('bench:'): - benchname = stat[6:] - if r.getstat(stat, None) is None: - row.append(('XXX',-1)) - elif relto: - factor = self.result(relto).getstat(stat)/r.getstat(stat) - if not r.asc_goods[benchname]: - factor = 1/factor - s, f = r.fmtstat(stat) - row.append((s + ' (%6.2fx)'%factor, f)) - else: - row.append(r.fmtstat(stat)) - else: - row.append(r.fmtstat(stat)) - table.append(row) - widths = [0 for thing in stats] - for row in table: - for i, cell in enumerate(row): - widths[i] = max(len(cell[0]), widths[i]) - concretetable = [] - concreterow = [] - for w, cell in zip(widths, table[0]): - concreterow.append(cell[0].center(w)) - concretetable.append(' '.join(concreterow)) - for row in table[1:]: - concreterow = [] - for w, cell in zip(widths, row): - concreterow.append("%*s"%(cell[1]*w, cell[0])) - concretetable.append(' '.join(concreterow)) - return concretetable - -class BenchmarkResult(object): - IDS = {} - - def __init__(self, exe, max_results=10): - self.max_results = max_results - self.exe_stat = os.stat(exe) - self.exe_name = exe - self.codesize = os.popen('size "%s" | tail -n1 | cut -f1'%(exe,)).read().strip() - try: - self.pypy_rev = int(os.popen( - exe + ' -c "import sys; print sys.pypy_version_info[-1]" 2>/dev/null').read().strip()) - except ValueError: - self.pypy_rev = -1 - self.best_benchmarks = {} - self.benchmarks = {} - self.asc_goods = {} - self.run_counts = {} - - def run_benchmark(self, benchmark, verbose=False): - self.asc_goods[benchmark.name] = benchmark.asc_good - if self.run_counts.get(benchmark.name, 0) > self.max_results: - return -1 - print 'running', benchmark.name, 'for', self.exe_name, - if verbose and self.pypy_rev > 0: - print '[rev %d]' % self.pypy_rev, - sys.stdout.flush() - new_result = benchmark.run(self.exe_name) - print new_result - if verbose: - print '{' - lines = benchmark.latest_output.splitlines(False) - for line in lines[:80]: - print '\t' + line - if len(lines) > 80: - print '\t....' - print '}' - self.run_counts[benchmark.name] = self.run_counts.get(benchmark.name, 0) + 1 - if new_result == '-FAILED-': - return 0 - self.benchmarks.setdefault(benchmark.name, []).append(new_result) - if benchmark.name in self.best_benchmarks: - old_result = self.best_benchmarks[benchmark.name] - if benchmark.asc_good: - new_result = max(new_result, old_result) - else: - new_result = min(new_result, old_result) - self.best_benchmarks[benchmark.name] = new_result - return 1 - - def getstat(self, *args): - # oh for supplied-p! - return_default = False - if len(args) == 1: - stat, = args - else: - stat, default = args - return_default = True - if hasattr(self, stat): - return getattr(self, stat) - if stat == 'exe': - myid = len(BenchmarkResult.IDS) - myid = BenchmarkResult.IDS.setdefault(self, myid) - return '[%s]' % myid - statkind, statdetail = stat.split(':') - if statkind == 'stat': - return getattr(self.exe_stat, statdetail) - elif statkind == 'bench': - if return_default: - return self.best_benchmarks.get(statdetail, default) - else: - return self.best_benchmarks[statdetail] - else: - 1/0 - - def fmtstat(self, *args): - stat = args[0] - statvalue = self.getstat(*args) - if stat == 'stat:st_mtime': - return time.ctime(statvalue), -1 - elif stat == 'exe_name': - return os.path.basename(statvalue), -1 - elif stat.startswith('bench:'): - from pypy.translator.benchmark import benchmarks - statkind, statdetail = stat.split(':', 1) - if '*' in statdetail: - statdetail = statdetail.split('*')[0] - b = benchmarks.BENCHMARKS_BY_NAME[statdetail] - return "%8.2f%s"%(statvalue, b.units), 1 - elif stat == 'pypy_rev': - return str(statvalue), 1 - else: - return str(statvalue), -1 - - def summary(self, stats): - return [self.getstat(stat) for stat in stats] - - def is_stable(self, name): - try: - return self.n_results[name] >= self.max_results - except: - return False - -if __name__ == '__main__': - import autopath - from pypy.translator.benchmark import benchmarks, result - import cPickle - if os.path.exists('foo.pickle'): - s = cPickle.load(open('foo.pickle', 'rb')) - else: - s = result.BenchmarkResultSet(4) - for exe in sys.argv[1:]: - r = s.result(exe) - r.run_benchmark(benchmarks.BENCHMARKS_BY_NAME['richards']) - r.run_benchmark(benchmarks.BENCHMARKS_BY_NAME['pystone']) - cPickle.dump(s, open('foo.pickle', 'wb')) - stats = ['stat:st_mtime', 'exe_name', 'bench:richards', 'bench:pystone'] - - for row in s.txt_summary(stats, sortby="exe_name", reverse=True, relto="/usr/local/bin/python2.4"): - print row --- a/pypy/translator/benchmark/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -import py - -def pytest_ignore_collect(path): - return path.basename == "test" --- a/pypy/translator/benchmark/jitbench.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys, os -from optparse import OptionParser - -parser = OptionParser() -parser.add_option( - '--size-factor-list', dest='sizefactorlist', - default='1,2,5,20,1,2,5,20,1,2,5,20', - ) -options, args = parser.parse_args(sys.argv[1:]) -args = args or [sys.executable] -executables = [os.path.abspath(executable) for executable in args] -sizefactors = [int(s) for s in options.sizefactorlist.split(',')] - -os.chdir(os.path.dirname(sys.argv[0]) or '.') - -errors = [] - -for sizefactor in sizefactors: - for executable in executables: - sys.argv[1:] = [executable, '--pickle=jitbench.benchmark_result', - '-v', '--no-cpython', - '--size-factor=%d' % sizefactor] - try: - execfile('bench-custom.py') - except SystemExit, e: - errors.append('%s:*%s: %s' % (executable, sizefactor, e)) - -if errors: - print '\n'.join(errors) - sys.exit(1) From commits-noreply at bitbucket.org Tue Dec 14 11:47:23 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 04:47:23 -0600 (CST) Subject: [pypy-svn] pypy commit 594f56f4cede: move also this conftest, else py.test does not find it Message-ID: <20101214104723.B71901E14AC@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292323289 -3600 # Node ID 594f56f4cede7c763dca05d1ac0693f40a9b76d0 # Parent a1d660058003b0c82e864c81be80b8093855cfc0 move also this conftest, else py.test does not find it --- a/pypy/module/cpyext/test/conftest.py +++ /dev/null @@ -1,16 +0,0 @@ -import py -from pypy.conftest import option, gettestobjspace - -def pytest_collect_directory(parent): - if parent.config.option.runappdirect: - py.test.skip("cannot be run by py.test -A") - - # ensure additional functions are registered - import pypy.module.cpyext.test.test_cpyext - -def pytest_funcarg__space(request): - return gettestobjspace(usemodules=['cpyext', 'thread']) - -def pytest_funcarg__api(request): - return request.cls.api - --- /dev/null +++ b/pypy/module/cpyext/conftest.py @@ -0,0 +1,16 @@ +import py +from pypy.conftest import option, gettestobjspace + +def pytest_collect_directory(parent): + if parent.config.option.runappdirect: + py.test.skip("cannot be run by py.test -A") + + # ensure additional functions are registered + import pypy.module.cpyext.test.test_cpyext + +def pytest_funcarg__space(request): + return gettestobjspace(usemodules=['cpyext', 'thread']) + +def pytest_funcarg__api(request): + return request.cls.api + From commits-noreply at bitbucket.org Tue Dec 14 12:00:13 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 05:00:13 -0600 (CST) Subject: [pypy-svn] pypy commit 8c19e0ff3e45: Fix the tests (sorry for the breakage). Message-ID: <20101214110013.718881E1071@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Armin Rigo # Date 1292324399 -3600 # Node ID 8c19e0ff3e45f1830755d008b19c680d88e7d714 # Parent 58f56bdf5c4bf4ca311b0f6a794ed8a79a1a85a8 Fix the tests (sorry for the breakage). --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -84,7 +84,7 @@ class Assembler386(object): self.fail_boxes_count = 0 self._current_depths_cache = (0, 0) self.datablockwrapper = None - self.stack_check_slowpath_imm = imm0 + self.stack_check_slowpath = 0 self.teardown() def leave_jitted_hook(self): @@ -196,10 +196,12 @@ class Assembler386(object): rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.malloc_fixedsize_slowpath2 = rawstart - _STACK_CHECK_SLOWPATH = lltype.Ptr(lltype.FuncType([lltype.Signed], - lltype.Void)) def _build_stack_check_slowpath(self): from pypy.rlib import rstack + _, _, slowpathaddr = self.cpu.insert_stack_check() + if slowpathaddr == 0 or self.cpu.exit_frame_with_exception_v < 0: + return # no stack check (for tests, or non-translated) + # mc = codebuf.MachineCodeBlockWrapper() mc.PUSH_r(ebp.value) mc.MOV_rr(ebp.value, esp.value) @@ -220,9 +222,7 @@ class Assembler386(object): mc.LEA_rb(edi.value, +16) mc.AND_ri(esp.value, -16) # - f = llhelper(self._STACK_CHECK_SLOWPATH, rstack.stack_check_slowpath) - addr = rffi.cast(lltype.Signed, f) - mc.CALL(imm(addr)) + mc.CALL(imm(slowpathaddr)) # mc.MOV(eax, heap(self.cpu.pos_exception())) mc.TEST_rr(eax.value, eax.value) @@ -257,7 +257,6 @@ class Assembler386(object): addr = self.cpu.get_on_leave_jitted_int(save_exception=False) mc.CALL(imm(addr)) # - assert self.cpu.exit_frame_with_exception_v >= 0 mc.MOV_ri(eax.value, self.cpu.exit_frame_with_exception_v) # # footer -- note the ADD, which skips the return address of this @@ -270,7 +269,7 @@ class Assembler386(object): mc.RET() # rawstart = mc.materialize(self.cpu.asmmemmgr, []) - self.stack_check_slowpath_imm = imm(rawstart) + self.stack_check_slowpath = rawstart def assemble_loop(self, inputargs, operations, looptoken, log): '''adds the following attributes to looptoken: @@ -547,16 +546,16 @@ class Assembler386(object): self.mc.PUSH_r(regloc.value) def _call_header_with_stack_check(self): - startaddr, length, slowpathaddr = self.cpu.insert_stack_check() - if slowpathaddr == 0: + if self.stack_check_slowpath == 0: pass # no stack check (e.g. not translated) else: + startaddr, length, _ = self.cpu.insert_stack_check() self.mc.MOV(eax, esp) # MOV eax, current self.mc.SUB(eax, heap(startaddr)) # SUB eax, [startaddr] self.mc.CMP(eax, imm(length)) # CMP eax, length self.mc.J_il8(rx86.Conditions['B'], 0) # JB .skip jb_location = self.mc.get_relative_pos() - self.mc.CALL(self.stack_check_slowpath_imm) # CALL slowpath + self.mc.CALL(imm(self.stack_check_slowpath))# CALL slowpath # patch the JB above # .skip: offset = self.mc.get_relative_pos() - jb_location assert 0 < offset <= 127 From commits-noreply at bitbucket.org Tue Dec 14 12:00:54 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 05:00:54 -0600 (CST) Subject: [pypy-svn] pypy commit 63d2c6f712c2: move this conftest too Message-ID: <20101214110054.43B50241316@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292324391 -3600 # Node ID 63d2c6f712c224253be400c4e41db00657350b73 # Parent 58f56bdf5c4bf4ca311b0f6a794ed8a79a1a85a8 move this conftest too --- a/pypy/jit/backend/cli/test/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -import py - -def pytest_collect_directory(path): - py.test.skip("CLI backend tests skipped for now") --- /dev/null +++ b/pypy/jit/backend/cli/conftest.py @@ -0,0 +1,4 @@ +import py + +def pytest_collect_directory(path): + py.test.skip("CLI backend tests skipped for now") From commits-noreply at bitbucket.org Tue Dec 14 12:00:54 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 05:00:54 -0600 (CST) Subject: [pypy-svn] pypy commit 51b2dc8ba40b: merge heads Message-ID: <20101214110054.EB1DF241416@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292324441 -3600 # Node ID 51b2dc8ba40bcb2ae4cc9a23e5016e3c928c595d # Parent 63d2c6f712c224253be400c4e41db00657350b73 # Parent 8c19e0ff3e45f1830755d008b19c680d88e7d714 merge heads From commits-noreply at bitbucket.org Tue Dec 14 13:23:43 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 06:23:43 -0600 (CST) Subject: [pypy-svn] pypy commit 987ea90229b5: Merge branch/getopt-appmain: refactor the option parsing done by appmain.py, Message-ID: <20101214122343.4A7C5241418@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Armin Rigo # Date 1292329367 -3600 # Node ID 987ea90229b55267e39fda11f2933dce10182a12 # Parent 51b2dc8ba40bcb2ae4cc9a23e5016e3c928c595d # Parent e73c414c718679a911fb725458acbc50b3d35640 Merge branch/getopt-appmain: refactor the option parsing done by appmain.py, to be more compatible with CPython's. From commits-noreply at bitbucket.org Tue Dec 14 13:30:00 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 06:30:00 -0600 (CST) Subject: [pypy-svn] pypy commit bec10379e55c: Remove merged branch. Message-ID: <20101214123000.38F4524142A@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Armin Rigo # Date 1292329789 -3600 # Node ID bec10379e55cbf8fa42fd58dbfd4030f55b12a13 # Parent e73c414c718679a911fb725458acbc50b3d35640 Remove merged branch. From commits-noreply at bitbucket.org Tue Dec 14 13:50:18 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 06:50:18 -0600 (CST) Subject: [pypy-svn] pypy commit 4959e3dccca7: Fix various tests crashing because of my recent changes at rlib.rstack. Message-ID: <20101214125018.80147241085@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Armin Rigo # Date 1292331005 -3600 # Node ID 4959e3dccca7ab37617ae8532e90cc6cb94efd72 # Parent 987ea90229b55267e39fda11f2933dce10182a12 Fix various tests crashing because of my recent changes at rlib.rstack. --- a/pypy/rpython/llinterp.py +++ b/pypy/rpython/llinterp.py @@ -948,6 +948,9 @@ class LLFrame(object): def op_set_stack_depth_limit(self): raise NotImplementedError("set_stack_depth_limit") + def op_stack_current(self): + return 0 + # operations on pyobjects! for opname in lloperation.opimpls.keys(): exec py.code.Source(""" --- a/pypy/translator/c/test/test_stackless.py +++ b/pypy/translator/c/test/test_stackless.py @@ -3,7 +3,7 @@ from pypy.translator.backendopt.all impo from pypy.translator.c.genc import CStandaloneBuilder from pypy.translator.c import gc from pypy.annotation.listdef import s_list_of_strings -from pypy.rlib.rstack import stack_unwind, stack_frames_depth, stack_too_big +from pypy.rlib.rstack import stack_unwind, stack_frames_depth from pypy.rlib.rstack import yield_current_frame_to_caller, set_stack_depth_limit from pypy.config.config import Config import os --- a/pypy/rlib/rstack.py +++ b/pypy/rlib/rstack.py @@ -37,14 +37,18 @@ def stack_frames_depth(): compilation_info = ExternalCompilationInfo(includes=['src/stack.h']) -def llexternal(name, args, res): +def llexternal(name, args, res, _callable=None): return rffi.llexternal(name, args, res, compilation_info=compilation_info, - sandboxsafe=True, _nowrapper=True) + sandboxsafe=True, _nowrapper=True, + _callable=_callable) -_stack_get_start = llexternal('LL_stack_get_start', [], lltype.Signed) -_stack_get_length = llexternal('LL_stack_get_length', [], lltype.Signed) +_stack_get_start = llexternal('LL_stack_get_start', [], lltype.Signed, + lambda: 0) +_stack_get_length = llexternal('LL_stack_get_length', [], lltype.Signed, + lambda: 1) _stack_too_big_slowpath = llexternal('LL_stack_too_big_slowpath', - [lltype.Signed], lltype.Char) + [lltype.Signed], lltype.Char, + lambda cur: '\x00') # the following is used by the JIT _stack_get_start_adr = llexternal('LL_stack_get_start_adr', [], lltype.Signed) From commits-noreply at bitbucket.org Tue Dec 14 13:59:48 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 06:59:48 -0600 (CST) Subject: [pypy-svn] pypy commit f40c2a8d81d8: Fix the test. Message-ID: <20101214125948.D1D87241085@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Armin Rigo # Date 1292331576 -3600 # Node ID f40c2a8d81d870e5b5be81dab104e93a05aedcbf # Parent 4959e3dccca7ab37617ae8532e90cc6cb94efd72 Fix the test. --- a/pypy/rlib/parsing/test/test_pythonparse.py +++ b/pypy/rlib/parsing/test/test_pythonparse.py @@ -240,7 +240,10 @@ a = 1 - 2 - 3 t = self.ToAST.transform(t) def test_parse_this(self): - s = py.path.local(__file__).read() + filename = __file__ + if filename.lower().endswith('.pyc'): + filename = filename[:-1] + s = py.path.local(filename).read() t = self.parse(s) t = self.ToAST.transform(t) From commits-noreply at bitbucket.org Tue Dec 14 14:03:14 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 07:03:14 -0600 (CST) Subject: [pypy-svn] pypy commit 3ff43a07d8ba: fix udir to use the mercurial info instead of the svn ones Message-ID: <20101214130314.DBA201E1071@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292331723 -3600 # Node ID 3ff43a07d8baa719585c39e09de341984c53dd86 # Parent 48b8bfd4c6913647ea4ed8281ae5b9c851bebdc9 fix udir to use the mercurial info instead of the svn ones --- a/pypy/tool/test/test_udir.py +++ b/pypy/tool/test/test_udir.py @@ -1,12 +1,6 @@ from pypy.tool import udir -def test_svn_info(): - res = udir.svn_info('http://codespeak.net/svn/pypy/dist/pypy/tool') - assert res == 'dist' - res = udir.svn_info('http://codespeak.net/svn/pypy/branch/stuff/pypy/tool') - assert res == 'stuff' - def test_make_udir(): root = str(udir.udir.ensure('make_udir1', dir=1)) p1 = udir.make_udir(dir=root) @@ -18,7 +12,6 @@ def test_make_udir(): def test_make_udir_with_basename(): root = str(udir.udir.ensure('make_udir2', dir=1)) - import pdb;pdb.set_trace() p1 = udir.make_udir(dir=root, basename='foobar') assert p1.relto(root) == 'usession-foobar-0' p1 = udir.make_udir(dir=root, basename='-foobar') --- a/pypy/tool/udir.py +++ b/pypy/tool/udir.py @@ -21,27 +21,22 @@ import autopath import os, sys import py +from pypy.tool.version import get_mercurial_info from py.path import local -def svn_info(url): - basename = url[:-len('pypy/tool')] - if basename.endswith('dist/'): - return 'dist' - else: - return basename.split('/')[-2] - PYPY_KEEP = int(os.environ.get('PYPY_USESSION_KEEP', '3')) def make_udir(dir=None, basename=None): if dir is not None: dir = local(dir) if basename is None: - try: - p = py.path.local(__file__).dirpath() - basename = svn_info(py.path.svnwc(p).info().url) + info = get_mercurial_info() + if info: + project, hgtag, hgid = info + basename = hgtag if isinstance(basename, unicode): basename = basename.encode(sys.getdefaultencoding()) - except: + else: basename = '' if not basename.startswith('-'): basename = '-' + basename From commits-noreply at bitbucket.org Tue Dec 14 14:03:14 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 07:03:14 -0600 (CST) Subject: [pypy-svn] pypy commit 48b8bfd4c691: write a (limited) test for sys._mercurial, and move the logic into tool so that it can be used elsewhere Message-ID: <20101214130314.561461E106D@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292330975 -3600 # Node ID 48b8bfd4c6913647ea4ed8281ae5b9c851bebdc9 # Parent 51b2dc8ba40bcb2ae4cc9a23e5016e3c928c595d write a (limited) test for sys._mercurial, and move the logic into tool so that it can be used elsewhere --- a/pypy/tool/test/test_udir.py +++ b/pypy/tool/test/test_udir.py @@ -8,7 +8,7 @@ def test_svn_info(): assert res == 'stuff' def test_make_udir(): - root = str(udir.udir.ensure('make_udir', dir=1)) + root = str(udir.udir.ensure('make_udir1', dir=1)) p1 = udir.make_udir(dir=root) p2 = udir.make_udir(dir=root) assert p1.relto(root).startswith('usession-') @@ -17,7 +17,8 @@ def test_make_udir(): assert p2.basename.endswith('-1') def test_make_udir_with_basename(): - root = str(udir.udir.ensure('make_udir', dir=1)) + root = str(udir.udir.ensure('make_udir2', dir=1)) + import pdb;pdb.set_trace() p1 = udir.make_udir(dir=root, basename='foobar') assert p1.relto(root) == 'usession-foobar-0' p1 = udir.make_udir(dir=root, basename='-foobar') --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -59,7 +59,7 @@ class Module(MixedModule): 'pypy_version_info' : 'version.get_pypy_version_info(space)', 'pypy_svn_url' : 'version.get_svn_url(space)', 'subversion' : 'version.get_subversion_info(space)', - '_mercurial' : 'version.get_mercurial_info(space)', + '_mercurial' : 'version.wrap_mercurial_info(space)', 'hexversion' : 'version.get_hexversion(space)', 'displayhook' : 'hook.displayhook', --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -32,6 +32,7 @@ def rev2int(rev): import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) del pypy +from pypy.tool.version import get_mercurial_info import time as t gmtime = t.gmtime() @@ -84,39 +85,18 @@ def get_subversion_info(space): space.wrap(svnbranch), space.wrap(str(svn_revision()))]) -def get_mercurial_info(space): - '''Obtain Mercurial version information by invoking the 'hg' command.''' - # TODO: support extracting from .hg_archival.txt - import py - from subprocess import Popen, PIPE - pypyroot = os.path.abspath(os.path.join(pypydir, '..')) - hgexe = py.path.local.sysfind('hg') - - if hgexe and os.path.isdir(os.path.join(pypyroot, '.hg')): - env = dict(os.environ) - # get Mercurial into scripting mode - env['HGPLAIN'] = '1' - # disable user configuration, extensions, etc. - env['HGRCPATH'] = os.devnull - - p = Popen([str(hgexe), 'id', '-i', pypyroot], stdout=PIPE, env=env) - hgid = p.stdout.read().strip() - - p = Popen([str(hgexe), 'id', '-t', pypyroot], stdout=PIPE, env=env) - hgtag = p.stdout.read().strip().split()[0] - - if hgtag == 'tip': - # use the branch instead - p = Popen([str(hgexe), 'id', '-b', pypyroot], stdout=PIPE, env=env) - hgtag = p.stdout.read().strip() - - return space.newtuple([space.wrap('PyPy'), +def wrap_mercurial_info(space): + info = get_mercurial_info() + if info: + project, hgtag, hgid = info + return space.newtuple([space.wrap(project), space.wrap(hgtag), space.wrap(hgid)]) else: return space.w_None + def tuple2hex(ver): d = {'alpha': 0xA, 'beta': 0xB, --- a/pypy/module/sys/test/test_sysmodule.py +++ b/pypy/module/sys/test/test_sysmodule.py @@ -462,6 +462,15 @@ class AppTestSysModulePortedFromCPython: assert svnbranch == svnbranch.strip('/') assert revision.isdigit() + def test__mercurial(self): + info = sys._mercurial + print info + if info: + project, hgtag, hgid = info + assert project == 'PyPy' + assert hgtag # no clue how to check something more :-/ + assert hgid + def test_trace_exec_execfile(self): found = [] def do_tracing(f, *args): --- /dev/null +++ b/pypy/tool/version.py @@ -0,0 +1,35 @@ +import py +import os +from subprocess import Popen, PIPE +import pypy +pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) + + +def get_mercurial_info(): + '''Obtain Mercurial version information by invoking the 'hg' command.''' + # TODO: support extracting from .hg_archival.txt + + pypyroot = os.path.abspath(os.path.join(pypydir, '..')) + hgexe = py.path.local.sysfind('hg') + + if hgexe and os.path.isdir(os.path.join(pypyroot, '.hg')): + env = dict(os.environ) + # get Mercurial into scripting mode + env['HGPLAIN'] = '1' + # disable user configuration, extensions, etc. + env['HGRCPATH'] = os.devnull + + p = Popen([str(hgexe), 'id', '-i', pypyroot], stdout=PIPE, env=env) + hgid = p.stdout.read().strip() + + p = Popen([str(hgexe), 'id', '-t', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip().split()[0] + + if hgtag == 'tip': + # use the branch instead + p = Popen([str(hgexe), 'id', '-b', pypyroot], stdout=PIPE, env=env) + hgtag = p.stdout.read().strip() + + return 'PyPy', hgtag, hgid + else: + return None From commits-noreply at bitbucket.org Tue Dec 14 14:03:15 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 07:03:15 -0600 (CST) Subject: [pypy-svn] pypy commit 8d549ed5ab18: merge heads Message-ID: <20101214130315.827161E14A3@bitbucket02.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292331779 -3600 # Node ID 8d549ed5ab18b56cf2bb0707e71a1c7d3eca0920 # Parent 3ff43a07d8baa719585c39e09de341984c53dd86 # Parent f40c2a8d81d870e5b5be81dab104e93a05aedcbf merge heads From commits-noreply at bitbucket.org Tue Dec 14 14:41:07 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 07:41:07 -0600 (CST) Subject: [pypy-svn] buildbot commit 0a92169ea38d: Fix the ordering: hg revisions are always more recent than svn ones. Message-ID: <20101214134107.85B016C1319@bitbucket03.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project buildbot # URL http://bitbucket.org/pypy/buildbot/overview # User Armin Rigo # Date 1292333974 -3600 # Node ID 0a92169ea38de840cee59fac65cf1d4255be0bda # Parent ebd81ebe66caa7a397b70e8daf588e90ca854f02 Fix the ordering: hg revisions are always more recent than svn ones. --- a/bot2/pypybuildbot/summary.py +++ b/bot2/pypybuildbot/summary.py @@ -645,8 +645,24 @@ class Summary(HtmlResource): @staticmethod def _prune_runs(runs, cutnum): + # + def revkey(rev): + if isinstance(rev, tuple): + extra, rev = rev + else: + extra = None + # subversion: just an integer + if isinstance(rev, int) or rev.isdigit(): + return (extra, 1, int(rev)) + # mercurial: "integer:globalid" + if ':' in rev and rev[:rev.index(':')].isdigit(): + i = rev.index(':') + return (extra, 2, int(rev[:i]), rev) + # unknown + return (extra, 3, rev) + # keys = runs.keys() - keys.sort() + keys.sort(key=revkey) if len(runs) > cutnum: for rev in keys[:-cutnum]: del runs[rev] --- a/bot2/pypybuildbot/test/test_summary.py +++ b/bot2/pypybuildbot/test/test_summary.py @@ -300,6 +300,30 @@ def test__prune_runs(): (4, 400): 40, (5, 20): 50 } + # with Mercurial, we get revision numbers of the form "localid:universalid" + # these ones should sort after the subversion numbers + runs = { + (1, "100"): 10, + (2, "200"): 20, + (3, "300"): 30, + (3, "2:b57f9a090b62"): 40, + (3, "10:34197134282a"): 45, + (5, "20"): 50 + } + summary.Summary._prune_runs(runs, 4) + assert len(runs) == 4 + assert runs == { + (3, "300"): 30, + (3, "2:b57f9a090b62"): 40, + (3, "10:34197134282a"): 45, + (5, "20"): 50 + } + summary.Summary._prune_runs(runs, 2) + assert len(runs) == 2 + assert runs == { + (3, "10:34197134282a"): 45, + (5, "20"): 50 + } def test_show_elapsed(): From cfbolz at codespeak.net Tue Dec 14 14:46:59 2010 From: cfbolz at codespeak.net (cfbolz at codespeak.net) Date: Tue, 14 Dec 2010 14:46:59 +0100 (CET) Subject: [pypy-svn] r80053 - pypy/extradoc/talk/pepm2011 Message-ID: <20101214134659.D045C50822@codespeak.net> Author: cfbolz Date: Tue Dec 14 14:46:55 2010 New Revision: 80053 Modified: pypy/extradoc/talk/pepm2011/escape-tracing.pdf pypy/extradoc/talk/pepm2011/math.lyx pypy/extradoc/talk/pepm2011/paper.tex Log: found an annoying typo in the paper, not that it helps now, but if we ever do something else with it, I want it documented Modified: pypy/extradoc/talk/pepm2011/escape-tracing.pdf ============================================================================== Binary files. No diff available. Modified: pypy/extradoc/talk/pepm2011/math.lyx ============================================================================== --- pypy/extradoc/talk/pepm2011/math.lyx (original) +++ pypy/extradoc/talk/pepm2011/math.lyx Tue Dec 14 14:46:55 2010 @@ -456,7 +456,7 @@ \begin_inset Text \begin_layout Plain Layout -\begin_inset Formula ${\displaystyle \frac{E(v)\notin\mathrm{dom}\left(S\right),\,\left(E(v),S\right)\overset{\mathrm{lift}}{\Longrightarrow}\left(\mathrm{ops},S^{\prime}\right)}{\mathtt{set}\left(v,F,u\right),E,S\overset{\mathrm{opt}}{\Longrightarrow}\mathrm{ops}::\left\langle \mathtt{set}\left(E(v),F,E(u)\right)\right\rangle ,E,S^{\prime}}}$ +\begin_inset Formula ${\displaystyle \frac{E(v)\notin\mathrm{dom}\left(S\right),\,\left(E(u),S\right)\overset{\mathrm{lift}}{\Longrightarrow}\left(\mathrm{ops},S^{\prime}\right)}{\mathtt{set}\left(v,F,u\right),E,S\overset{\mathrm{opt}}{\Longrightarrow}\mathrm{ops}::\left\langle \mathtt{set}\left(E(v),F,E(u)\right)\right\rangle ,E,S^{\prime}}}$ \end_inset Modified: pypy/extradoc/talk/pepm2011/paper.tex ============================================================================== --- pypy/extradoc/talk/pepm2011/paper.tex (original) +++ pypy/extradoc/talk/pepm2011/paper.tex Tue Dec 14 14:46:55 2010 @@ -827,7 +827,7 @@ \emph{get} & ${\displaystyle \frac{E(v)\in\mathrm{dom}(S)}{u=\mathtt{get}(v,F),E,S\overset{\mathrm{opt}}{\Longrightarrow}\left\langle \,\right\rangle ,E\left[u\mapsto S(E(v))_{F}\right],S}}$\tabularnewline[3em] & ${\displaystyle \frac{E(v)\notin\mathrm{dom}(S),\,\,\, u^{*}\,\mathrm{fresh}}{u=\mathtt{get}(v,F),E,S\overset{\mathrm{opt}}{\Longrightarrow}\left\langle u^{*}=\mathtt{get}(E(v),F)\right\rangle ,E\left[u\mapsto u^{*}\right],S}}$\tabularnewline[3em] \emph{set} & ${\displaystyle \frac{E(v)\in\mathrm{dom}(S)}{\mathtt{set}\left(v,F,u\right),E,S\overset{\mathrm{opt}}{\Longrightarrow}\left\langle \,\right\rangle ,E,S\left[E\left(v\right)\mapsto\left(S(E(v))!_{F}E(u)\right)\right]}}$\tabularnewline[3em] - & ${\displaystyle \frac{E(v)\notin\mathrm{dom}\left(S\right),\,\left(E(v),S\right)\overset{\mathrm{lift}}{\Longrightarrow}\left(\mathrm{ops},S^{\prime}\right)}{\mathtt{set}\left(v,F,u\right),E,S\overset{\mathrm{opt}}{\Longrightarrow}\mathrm{ops}::\left\langle \mathtt{set}\left(E(v),F,E(u)\right)\right\rangle ,E,S^{\prime}}}$\tabularnewline[3em] + & ${\displaystyle \frac{E(v)\notin\mathrm{dom}\left(S\right),\,\left(E(u),S\right)\overset{\mathrm{lift}}{\Longrightarrow}\left(\mathrm{ops},S^{\prime}\right)}{\mathtt{set}\left(v,F,u\right),E,S\overset{\mathrm{opt}}{\Longrightarrow}\mathrm{ops}::\left\langle \mathtt{set}\left(E(v),F,E(u)\right)\right\rangle ,E,S^{\prime}}}$\tabularnewline[3em] \emph{guard} & ${\displaystyle \frac{E(v)\in\mathrm{dom}(S),\,\mathrm{type}(S(E(v)))=T}{\mathtt{guard\_class}(v,T),E,S\overset{\mathrm{opt}}{\Longrightarrow}\left\langle \,\right\rangle ,E,S}}$\tabularnewline[3em] & ${\displaystyle \frac{E(v)\notin\mathrm{dom}(S)\vee\mathrm{type}(S(E(v)))\neq T,\,\left(E(v),S\right)\overset{\mathrm{lift}}{\Longrightarrow}\left(\mathrm{ops},S^{\prime}\right)}{\mathtt{guard\_class}(v,T),E,S\overset{\mathrm{opt}}{\Longrightarrow}\mathrm{ops}::\left\langle \mathtt{guard\_class}(E\left(v\right),T)\right\rangle ,E,S^{\prime}}}$\tabularnewline[3em] \emph{lifting} & ${\displaystyle \frac{v^{*}\notin\mathrm{dom}(S)}{v^{*},S\overset{\mathrm{lift}}{\Longrightarrow}\left\langle \,\right\rangle ,S}}$\tabularnewline[3em] From commits-noreply at bitbucket.org Tue Dec 14 14:50:55 2010 From: commits-noreply at bitbucket.org (Bitbucket) Date: Tue, 14 Dec 2010 07:50:55 -0600 (CST) Subject: [pypy-svn] pypy commit 11dd42c70507: yet another approach for skipping whole directories, hopefully this one will work Message-ID: <20101214135055.56705241416@bitbucket01.managed.contegix.com> # HG changeset patch -- Bitbucket.org # Project pypy # URL http://bitbucket.org/pypy/pypy/overview # User Antonio Cuni # Date 1292334629 -3600 # Node ID 11dd42c70507da9dc1ae3c2692f4ccaa4b3ade90 # Parent 8d549ed5ab18b56cf2bb0707e71a1c7d3eca0920 yet another approach for skipping whole directories, hopefully this one will work --- /dev/null +++ b/pypy/jit/backend/cli/test/conftest.py @@ -0,0 +1,4 @@ +import py + +def pytest_ignore_collect(path): + return True --- a/pypy/rlib/rsdl/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -from pypy.rlib.rsdl.eci import check_sdl_installation, SDLNotInstalled -import py - -def pytest_collect_directory(): - try: - check_sdl_installation() - except SDLNotInstalled, e: - py.test.skip("SDL not installed(?): %s" % (e,)) --- /dev/null +++ b/pypy/jit/backend/llvm/test/conftest.py @@ -0,0 +1,4 @@ +import py + +def pytest_ignore_collect(path): + return True --- /dev/null +++ b/pypy/rlib/rsdl/test/conftest.py @@ -0,0 +1,10 @@ +from pypy.rlib.rsdl.eci import check_sdl_installation, SDLNotInstalled +import py + +def pytest_ignore_collect(path): + try: + check_sdl_installation() + except SDLNotInstalled, e: + return True + else: + return False --- a/pypy/jit/backend/cli/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -import py - -def pytest_collect_directory(path): - py.test.skip("CLI backend tests skipped for now") --- a/pypy/jit/backend/llvm/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -import py - -def pytest_collect_directory(): - py.test.skip("llvm backend tests skipped for now") From fijal at codespeak.net Tue Dec 14 16:38:58 2010 From: fijal at codespeak.net (fijal at codespeak.net) Date: Tue, 14 Dec 2010 16:38:58 +0100 (CET) Subject: [pypy-svn] r80054 - in pypy/extradoc/pypy.org: . source/_layouts Message-ID: <20101214153858.E365C50822@codespeak.net> Author: fijal Date: Tue Dec 14 16:38:56 2010 New Revision: 80054 Modified: pypy/extradoc/pypy.org/archive.html pypy/extradoc/pypy.org/compat.html pypy/extradoc/pypy.org/contact.html pypy/extradoc/pypy.org/download.html pypy/extradoc/pypy.org/features.html pypy/extradoc/pypy.org/howtohelp.html pypy/extradoc/pypy.org/index.html pypy/extradoc/pypy.org/source/_layouts/site.genshi Log: change svn logo to hg logo Modified: pypy/extradoc/pypy.org/archive.html ============================================================================== --- pypy/extradoc/pypy.org/archive.html (original) +++ pypy/extradoc/pypy.org/archive.html Tue Dec 14 16:38:56 2010 @@ -30,7 +30,7 @@